mmalygina
/
xk6-frostfs
forked from TrueCloudLab/xk6-frostfs
1
0
Fork 0
Code Pull Requests Activity

[#119] metrics: Refactor custom metrics 

Add `data` metrics to measure payload rate.
Rename `total` metrics to `success`, because these metrics count
success operations count but not total operations count.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
feauture-add-s3-dar-scenario
Dmitrii Stepanov 2024-01-24 12:44:16 +03:00
parent 604982de3e
commit e9edca3e79
7 changed files with 66 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
internal/local/local.go
View File

@ -60,9 +60,9 @@ var (
_ modules.Module = &RootModule{}
_ modules.Instance = &Local{}
objPutTotal, objPutFails, objPutDuration *metrics.Metric
objGetTotal, objGetFails, objGetDuration *metrics.Metric
objDeleteTotal, objDeleteFails, objDeleteDuration *metrics.Metric
objPutSuccess, objPutFails, objPutDuration, objPutData *metrics.Metric
objGetSuccess, objGetFails, objGetDuration, objGetData *metrics.Metric
objDeleteSuccess, objDeleteFails, objDeleteDuration *metrics.Metric
)
func init() {
@ -173,15 +173,17 @@ func (s *Local) Connect(configFile, configDir, hexKey string, debug bool, maxSiz
}
// Register metrics.
objPutTotal, _ = stats.Registry.NewMetric("local_obj_put_total", metrics.Counter)
objPutSuccess, _ = stats.Registry.NewMetric("local_obj_put_success", metrics.Counter)
objPutFails, _ = stats.Registry.NewMetric("local_obj_put_fails", metrics.Counter)
objPutDuration, _ = stats.Registry.NewMetric("local_obj_put_duration", metrics.Trend, metrics.Time)
objPutData, _ = stats.Registry.NewMetric("local_obj_put_bytes", metrics.Counter, metrics.Data)
objGetTotal, _ = stats.Registry.NewMetric("local_obj_get_total", metrics.Counter)
objGetSuccess, _ = stats.Registry.NewMetric("local_obj_get_success", metrics.Counter)
objGetFails, _ = stats.Registry.NewMetric("local_obj_get_fails", metrics.Counter)
objGetDuration, _ = stats.Registry.NewMetric("local_obj_get_duration", metrics.Trend, metrics.Time)
objGetData, _ = stats.Registry.NewMetric("local_obj_get_bytes", metrics.Counter, metrics.Data)
objDeleteTotal, _ = stats.Registry.NewMetric("local_obj_delete_total", metrics.Counter)
objDeleteSuccess, _ = stats.Registry.NewMetric("local_obj_delete_success", metrics.Counter)
objDeleteFails, _ = stats.Registry.NewMetric("local_obj_delete_fails", metrics.Counter)
objDeleteDuration, _ = stats.Registry.NewMetric("local_obj_delete_duration", metrics.Trend, metrics.Time)
@ -192,25 +194,27 @@ func (s *Local) Connect(configFile, configDir, hexKey string, debug bool, maxSiz
if err != nil {
stats.Report(s.vu, objPutFails, 1)
} else {
stats.Report(s.vu, objPutTotal, 1)
stats.Report(s.vu, objPutSuccess, 1)
stats.ReportDataSent(s.vu, float64(sz))
stats.Report(s.vu, objPutDuration, metrics.D(dt))
stats.Report(s.vu, objPutData, float64(sz))
}
}),
rawclient.WithGetHandler(func(sz uint64, err error, dt time.Duration) {
if err != nil {
stats.Report(s.vu, objGetFails, 1)
} else {
stats.Report(s.vu, objGetTotal, 1)
stats.Report(s.vu, objGetSuccess, 1)
stats.Report(s.vu, objGetDuration, metrics.D(dt))
stats.ReportDataReceived(s.vu, float64(sz))
stats.Report(s.vu, objGetData, float64(sz))
}
}),
rawclient.WithDeleteHandler(func(err error, dt time.Duration) {
if err != nil {
stats.Report(s.vu, objDeleteFails, 1)
} else {
stats.Report(s.vu, objDeleteTotal, 1)
stats.Report(s.vu, objDeleteSuccess, 1)
stats.Report(s.vu, objDeleteDuration, metrics.D(dt))
}
}),

8
internal/native/client.go
View File

@ -137,7 +137,7 @@ func (c *Client) Delete(containerID string, objectID string) DeleteResponse {
return DeleteResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, objDeleteTotal, 1)
stats.Report(c.vu, objDeleteSuccess, 1)
stats.Report(c.vu, objDeleteDuration, metrics.D(time.Since(start)))
return DeleteResponse{Success: true}
}
@ -171,9 +171,10 @@ func (c *Client) Get(containerID, objectID string) GetResponse {
return GetResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, objGetTotal, 1)
stats.Report(c.vu, objGetSuccess, 1)
stats.Report(c.vu, objGetDuration, metrics.D(time.Since(start)))
stats.ReportDataReceived(c.vu, float64(objSize))
stats.Report(c.vu, objGetData, float64(objSize))
return GetResponse{Success: true}
}
@ -468,9 +469,10 @@ func put(vu modules.VU, cli *client.Client, prepareLocally bool, tok *session.Ob
return nil, err
}
stats.Report(vu, objPutTotal, 1)
stats.Report(vu, objPutSuccess, 1)
stats.ReportDataSent(vu, float64(sz))
stats.Report(vu, objPutDuration, metrics.D(time.Since(start)))
stats.Report(vu, objPutData, float64(sz))
return resp, nil
}

16
internal/native/native.go
View File

@ -29,10 +29,10 @@ var (
_ modules.Instance = &Native{}
_ modules.Module = &RootModule{}
objPutTotal, objPutFails, objPutDuration *metrics.Metric
objGetTotal, objGetFails, objGetDuration *metrics.Metric
objDeleteTotal, objDeleteFails, objDeleteDuration *metrics.Metric
cnrPutTotal, cnrPutFails, cnrPutDuration *metrics.Metric
objPutSuccess, objPutFails, objPutDuration, objPutData *metrics.Metric
objGetSuccess, objGetFails, objGetDuration, objGetData *metrics.Metric
objDeleteSuccess, objDeleteFails, objDeleteDuration *metrics.Metric
cnrPutTotal, cnrPutFails, cnrPutDuration *metrics.Metric
)
func init() {
@ -117,15 +117,17 @@ func (n *Native) Connect(endpoint, hexPrivateKey string, dialTimeout, streamTime
// register metrics
objPutTotal, _ = stats.Registry.NewMetric("frostfs_obj_put_total", metrics.Counter)
objPutSuccess, _ = stats.Registry.NewMetric("frostfs_obj_put_success", metrics.Counter)
objPutFails, _ = stats.Registry.NewMetric("frostfs_obj_put_fails", metrics.Counter)
objPutDuration, _ = stats.Registry.NewMetric("frostfs_obj_put_duration", metrics.Trend, metrics.Time)
objPutData, _ = stats.Registry.NewMetric("frostfs_obj_put_bytes", metrics.Counter, metrics.Data)
objGetTotal, _ = stats.Registry.NewMetric("frostfs_obj_get_total", metrics.Counter)
objGetSuccess, _ = stats.Registry.NewMetric("frostfs_obj_get_success", metrics.Counter)
objGetFails, _ = stats.Registry.NewMetric("frostfs_obj_get_fails", metrics.Counter)
objGetDuration, _ = stats.Registry.NewMetric("frostfs_obj_get_duration", metrics.Trend, metrics.Time)
objGetData, _ = stats.Registry.NewMetric("frostfs_obj_get_bytes", metrics.Counter, metrics.Data)
objDeleteTotal, _ = stats.Registry.NewMetric("frostfs_obj_delete_total", metrics.Counter)
objDeleteSuccess, _ = stats.Registry.NewMetric("frostfs_obj_delete_success", metrics.Counter)
objDeleteFails, _ = stats.Registry.NewMetric("frostfs_obj_delete_fails", metrics.Counter)
objDeleteDuration, _ = stats.Registry.NewMetric("frostfs_obj_delete_duration", metrics.Trend, metrics.Time)

15
internal/s3/client.go
View File

@ -65,9 +65,10 @@ func (c *Client) Put(bucket, key string, payload datagen.Payload) PutResponse {
return PutResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, objPutTotal, 1)
stats.Report(c.vu, objPutSuccess, 1)
stats.ReportDataSent(c.vu, float64(sz))
stats.Report(c.vu, objPutDuration, metrics.D(time.Since(start)))
stats.Report(c.vu, objPutData, float64(sz))
return PutResponse{Success: true}
}
@ -98,9 +99,10 @@ func (c *Client) Multipart(bucket, key string, objPartSize, concurrency int, pay
return PutResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, objPutTotal, 1)
stats.Report(c.vu, objPutSuccess, 1)
stats.ReportDataSent(c.vu, float64(sz))
stats.Report(c.vu, objPutDuration, metrics.D(time.Since(start)))
stats.Report(c.vu, objPutData, float64(sz))
return PutResponse{Success: true}
}
@ -116,7 +118,7 @@ func (c *Client) Delete(bucket, key string) DeleteResponse {
return DeleteResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, objDeleteTotal, 1)
stats.Report(c.vu, objDeleteSuccess, 1)
stats.Report(c.vu, objDeleteDuration, metrics.D(time.Since(start)))
return DeleteResponse{Success: true}
}
@ -124,7 +126,7 @@ func (c *Client) Delete(bucket, key string) DeleteResponse {
func (c *Client) Get(bucket, key string) GetResponse {
start := time.Now()
var objSize = 0
objSize := 0
err := get(c.cli, bucket, key, func(chunk []byte) {
objSize += len(chunk)
})
@ -133,9 +135,10 @@ func (c *Client) Get(bucket, key string) GetResponse {
return GetResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, objGetTotal, 1)
stats.Report(c.vu, objGetSuccess, 1)
stats.Report(c.vu, objGetDuration, metrics.D(time.Since(start)))
stats.ReportDataReceived(c.vu, float64(objSize))
stats.Report(c.vu, objGetData, float64(objSize))
return GetResponse{Success: true}
}
@ -212,7 +215,7 @@ func (c *Client) CreateBucket(bucket string, params map[string]string) CreateBuc
return CreateBucketResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, createBucketTotal, 1)
stats.Report(c.vu, createBucketSuccess, 1)
stats.Report(c.vu, createBucketDuration, metrics.D(time.Since(start)))
return CreateBucketResponse{Success: true}
}

18
internal/s3/s3.go
View File

@ -29,10 +29,10 @@ var (
_ modules.Instance = &S3{}
_ modules.Module = &RootModule{}
objPutTotal, objPutFails, objPutDuration *metrics.Metric
objGetTotal, objGetFails, objGetDuration *metrics.Metric
objDeleteTotal, objDeleteFails, objDeleteDuration *metrics.Metric
createBucketTotal, createBucketFails, createBucketDuration *metrics.Metric
objPutSuccess, objPutFails, objPutDuration, objPutData *metrics.Metric
objGetSuccess, objGetFails, objGetDuration, objGetData *metrics.Metric
objDeleteSuccess, objDeleteFails, objDeleteDuration *metrics.Metric
createBucketSuccess, createBucketFails, createBucketDuration *metrics.Metric
)
func init() {
@ -95,19 +95,21 @@ func (s *S3) Connect(endpoint string, params map[string]string) (*Client, error)
})
// register metrics
objPutTotal, _ = stats.Registry.NewMetric("aws_obj_put_total", metrics.Counter)
objPutSuccess, _ = stats.Registry.NewMetric("aws_obj_put_success", metrics.Counter)
objPutFails, _ = stats.Registry.NewMetric("aws_obj_put_fails", metrics.Counter)
objPutDuration, _ = stats.Registry.NewMetric("aws_obj_put_duration", metrics.Trend, metrics.Time)
objPutData, _ = stats.Registry.NewMetric("aws_obj_put_bytes", metrics.Counter, metrics.Data)
objGetTotal, _ = stats.Registry.NewMetric("aws_obj_get_total", metrics.Counter)
objGetSuccess, _ = stats.Registry.NewMetric("aws_obj_get_success", metrics.Counter)
objGetFails, _ = stats.Registry.NewMetric("aws_obj_get_fails", metrics.Counter)
objGetDuration, _ = stats.Registry.NewMetric("aws_obj_get_duration", metrics.Trend, metrics.Time)
objGetData, _ = stats.Registry.NewMetric("aws_obj_get_bytes", metrics.Counter, metrics.Data)
objDeleteTotal, _ = stats.Registry.NewMetric("aws_obj_delete_total", metrics.Counter)
objDeleteSuccess, _ = stats.Registry.NewMetric("aws_obj_delete_success", metrics.Counter)
objDeleteFails, _ = stats.Registry.NewMetric("aws_obj_delete_fails", metrics.Counter)
objDeleteDuration, _ = stats.Registry.NewMetric("aws_obj_delete_duration", metrics.Trend, metrics.Time)
createBucketTotal, _ = stats.Registry.NewMetric("aws_create_bucket_total", metrics.Counter)
createBucketSuccess, _ = stats.Registry.NewMetric("aws_create_bucket_success", metrics.Counter)
createBucketFails, _ = stats.Registry.NewMetric("aws_create_bucket_fails", metrics.Counter)
createBucketDuration, _ = stats.Registry.NewMetric("aws_create_bucket_duration", metrics.Trend, metrics.Time)

6
internal/s3local/client.go
View File

@ -68,8 +68,9 @@ func (c *Client) Put(bucket, key string, payload datagen.Payload) PutResponse {
}
stats.Report(c.vu, objPutDuration, metrics.D(time.Since(start)))
stats.Report(c.vu, objPutTotal, 1)
stats.Report(c.vu, objPutSuccess, 1)
stats.ReportDataSent(c.vu, float64(prm.Size))
stats.Report(c.vu, objPutData, float64(prm.Size))
return PutResponse{Success: true}
}
@ -115,8 +116,9 @@ func (c *Client) Get(bucket, key string) GetResponse {
}
stats.Report(c.vu, objGetDuration, metrics.D(time.Since(start)))
stats.Report(c.vu, objGetTotal, 1)
stats.Report(c.vu, objGetSuccess, 1)
stats.ReportDataReceived(c.vu, wr.total)
stats.Report(c.vu, objGetData, wr.total)
return GetResponse{Success: true}
}

26
internal/s3local/local.go
View File

@ -32,10 +32,10 @@ var (
_ modules.Module = &RootModule{}
_ modules.Instance = &Local{}
internalObjPutTotal, internalObjPutFails, internalObjPutDuration *metrics.Metric
internalObjGetTotal, internalObjGetFails, internalObjGetDuration *metrics.Metric
objPutTotal, objPutFails, objPutDuration *metrics.Metric
objGetTotal, objGetFails, objGetDuration *metrics.Metric
internalObjPutSuccess, internalObjPutFails, internalObjPutDuration, internalObjPutData *metrics.Metric
internalObjGetSuccess, internalObjGetFails, internalObjGetDuration, internalObjGetData *metrics.Metric
objPutSuccess, objPutFails, objPutDuration, objPutData *metrics.Metric
objGetSuccess, objGetFails, objGetDuration, objGetData *metrics.Metric
)
func init() {
@ -88,21 +88,25 @@ func (s *Local) Connect(configFile string, configDir string, params map[string]s
}
// Register metrics.
internalObjPutTotal, _ = stats.Registry.NewMetric("s3local_internal_obj_put_total", metrics.Counter)
internalObjPutSuccess, _ = stats.Registry.NewMetric("s3local_internal_obj_put_success", metrics.Counter)
internalObjPutFails, _ = stats.Registry.NewMetric("s3local_internal_obj_put_fails", metrics.Counter)
internalObjPutDuration, _ = stats.Registry.NewMetric("s3local_internal_obj_put_duration", metrics.Trend, metrics.Time)
internalObjPutData, _ = stats.Registry.NewMetric("s3local_internal_obj_put_bytes", metrics.Counter, metrics.Data)
internalObjGetTotal, _ = stats.Registry.NewMetric("s3local_internal_obj_get_total", metrics.Counter)
internalObjGetSuccess, _ = stats.Registry.NewMetric("s3local_internal_obj_get_success", metrics.Counter)
internalObjGetFails, _ = stats.Registry.NewMetric("s3local_internal_obj_get_fails", metrics.Counter)
internalObjGetDuration, _ = stats.Registry.NewMetric("s3local_internal_obj_get_duration", metrics.Trend, metrics.Time)
internalObjGetData, _ = stats.Registry.NewMetric("s3local_internal_obj_get_bytes", metrics.Counter, metrics.Data)
objPutTotal, _ = stats.Registry.NewMetric("s3local_obj_put_total", metrics.Counter)
objPutSuccess, _ = stats.Registry.NewMetric("s3local_obj_put_success", metrics.Counter)
objPutFails, _ = stats.Registry.NewMetric("s3local_obj_put_fails", metrics.Counter)
objPutDuration, _ = stats.Registry.NewMetric("s3local_obj_put_duration", metrics.Trend, metrics.Time)
objPutData, _ = stats.Registry.NewMetric("s3local_obj_put_bytes", metrics.Counter, metrics.Data)
objGetTotal, _ = stats.Registry.NewMetric("s3local_obj_get_total", metrics.Counter)
objGetSuccess, _ = stats.Registry.NewMetric("s3local_obj_get_success", metrics.Counter)
objGetFails, _ = stats.Registry.NewMetric("s3local_obj_get_fails", metrics.Counter)
objGetDuration, _ = stats.Registry.NewMetric("s3local_obj_get_duration", metrics.Trend, metrics.Time)
objGetData, _ = stats.Registry.NewMetric("s3local_obj_get_bytes", metrics.Counter, metrics.Data)
// Create S3 layer backed by local storage engine and tree service.
ng, limiter, err := s.l.ResolveEngine(s.l.VU().Context(), configFile, configDir, *debugLogger, maxSizeGB)
@ -122,16 +126,18 @@ func (s *Local) Connect(configFile string, configDir string, params map[string]s
if err != nil {
stats.Report(s.l.VU(), internalObjPutFails, 1)
} else {
stats.Report(s.l.VU(), internalObjPutTotal, 1)
stats.Report(s.l.VU(), internalObjPutSuccess, 1)
stats.Report(s.l.VU(), internalObjPutDuration, metrics.D(dt))
stats.Report(s.l.VU(), internalObjPutData, float64(sz))
}
}),
rawclient.WithGetHandler(func(sz uint64, err error, dt time.Duration) {
if err != nil {
stats.Report(s.l.VU(), internalObjGetFails, 1)
} else {
stats.Report(s.l.VU(), internalObjGetTotal, 1)
stats.Report(s.l.VU(), internalObjGetSuccess, 1)
stats.Report(s.l.VU(), internalObjGetDuration, metrics.D(dt))
stats.Report(s.l.VU(), internalObjGetData, float64(sz))
}
}),
)