From 962da644af16f573aaac712971483f8f8497cd7c Mon Sep 17 00:00:00 2001 From: Vladimir Domnich Date: Fri, 2 Sep 2022 23:23:33 +0400 Subject: [PATCH] [#17] Implement k6 extension for random data generation It improves payload generation in our scenarios. Current implementation of scenarios generates single random payload at the start and then sends this same payload on every request. More realistic test is to generate unique payload for each request. However, this is an expensive operation that can easily cause a bottleneck on K6 side when we run multiple writing VUs. So instead we generate a random buffer with some extra bytes and then take slices of this buffer thus producing a random payload for each request. Signed-off-by: Vladimir Domnich --- .gitignore | 1 + internal/datagen/datagen.go | 42 +++++++++++++++++++ internal/datagen/generator.go | 77 +++++++++++++++++++++++++++++++++++ neofs.go | 1 + scenarios/grpc.js | 5 ++- scenarios/http.js | 5 ++- scenarios/s3.js | 6 ++- 7 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 .gitignore create mode 100644 internal/datagen/datagen.go create mode 100644 internal/datagen/generator.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5ed2f93 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +k6 diff --git a/internal/datagen/datagen.go b/internal/datagen/datagen.go new file mode 100644 index 0000000..8e8f3c9 --- /dev/null +++ b/internal/datagen/datagen.go @@ -0,0 +1,42 @@ +package datagen + +import ( + "go.k6.io/k6/js/modules" +) + +// RootModule is the global module object type. It is instantiated once per test +// run and will be used to create k6/x/neofs/registry module instances for each VU. +type RootModule struct{} + +// Datagen represents an instance of the module for every VU. +type Datagen struct { + vu modules.VU +} + +// Ensure the interfaces are implemented correctly. +var ( + _ modules.Instance = &Datagen{} + _ modules.Module = &RootModule{} +) + +func init() { + modules.Register("k6/x/neofs/datagen", new(RootModule)) +} + +// NewModuleInstance implements the modules.Module interface and returns +// a new instance for each VU. +func (r *RootModule) NewModuleInstance(vu modules.VU) modules.Instance { + mi := &Datagen{vu: vu} + return mi +} + +// Exports implements the modules.Instance interface and returns the exports +// of the JS module. +func (d *Datagen) Exports() modules.Exports { + return modules.Exports{Default: d} +} + +func (d *Datagen) Generator(size int) *Generator { + g := NewGenerator(d.vu, size) + return &g +} diff --git a/internal/datagen/generator.go b/internal/datagen/generator.go new file mode 100644 index 0000000..fc6699e --- /dev/null +++ b/internal/datagen/generator.go @@ -0,0 +1,77 @@ +package datagen + +import ( + "crypto/sha256" + "encoding/hex" + "math/rand" + "time" + + "github.com/dop251/goja" + "go.k6.io/k6/js/modules" +) + +type ( + // Generator stores buffer of random bytes with some tail and returns data slices with + // an increasing offset so that we receive a different set of bytes after each call without + // re-generation of the entire buffer from scratch: + // + // [<----------size----------><-tail->] + // [<----------slice0-------->........] + // [.<----------slice1-------->.......] + // [..<----------slice2-------->......] + Generator struct { + vu modules.VU + size int + buf []byte + offset int + } + + GenPayloadResponse struct { + Payload goja.ArrayBuffer + Hash string + } +) + +// TailSize specifies number of extra random bytes in the buffer tail. +const TailSize = 1024 + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func NewGenerator(vu modules.VU, size int) Generator { + return Generator{vu: vu, size: size, buf: nil, offset: 0} +} + +func (g *Generator) GenPayload(calcHash bool) GenPayloadResponse { + data := g.nextSlice() + + dataHash := "" + if calcHash { + hashBytes := sha256.Sum256(data) + dataHash = hex.EncodeToString(hashBytes[:]) + } + + payload := g.vu.Runtime().NewArrayBuffer(data) + return GenPayloadResponse{Payload: payload, Hash: dataHash} +} + +func (g *Generator) nextSlice() []byte { + if g.buf == nil { + // Allocate buffer with extra tail for sliding and populate it with random bytes + g.buf = make([]byte, g.size+TailSize) + rand.Read(g.buf) // Per docs, err is always nil here + } + + result := g.buf[g.offset : g.offset+g.size] + + // Shift the offset for the next call. If we've used our entire tail, then erase + // the buffer so that on the next call it is regenerated anew + g.offset += 1 + if g.offset >= TailSize { + g.buf = nil + g.offset = 0 + } + + return result +} diff --git a/neofs.go b/neofs.go index c6f6877..59af1d3 100644 --- a/neofs.go +++ b/neofs.go @@ -1,6 +1,7 @@ package xk6_neofs import ( + _ "github.com/nspcc-dev/xk6-neofs/internal/datagen" _ "github.com/nspcc-dev/xk6-neofs/internal/native" _ "github.com/nspcc-dev/xk6-neofs/internal/s3" "go.k6.io/k6/js/modules" diff --git a/scenarios/grpc.js b/scenarios/grpc.js index 11b8615..3acb2d4 100644 --- a/scenarios/grpc.js +++ b/scenarios/grpc.js @@ -1,5 +1,5 @@ +import datagen from 'k6/x/neofs/datagen'; import native from 'k6/x/neofs/native'; -import crypto from 'k6/crypto'; import { SharedArray } from 'k6/data'; import { sleep } from 'k6'; @@ -26,7 +26,7 @@ const [ write, duration ] = __ENV.PROFILE.split(':'); let vus_read = Math.ceil(__ENV.CLIENTS/100*(100-parseInt(write))) let vus_write = __ENV.CLIENTS - vus_read -const payload = crypto.randomBytes(1024*parseInt(__ENV.WRITE_OBJ_SIZE)) +const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE)); let nodes = __ENV.NODES.split(',') let rand_node = nodes[Math.floor(Math.random()*nodes.length)]; @@ -72,6 +72,7 @@ export function obj_write() { } let container = container_list[Math.floor(Math.random()*container_list.length)]; + const { payload } = generator.genPayload(false); let resp = neofs_cli.put( container, headers, payload); if (!resp.success) { console.log(resp.error); diff --git a/scenarios/http.js b/scenarios/http.js index bcd1ec8..42d2d47 100644 --- a/scenarios/http.js +++ b/scenarios/http.js @@ -1,5 +1,5 @@ +import datagen from 'k6/x/neofs/datagen'; import http from 'k6/http'; -import crypto from 'k6/crypto'; import { SharedArray } from 'k6/data'; import { sleep } from 'k6'; @@ -24,7 +24,7 @@ const [ write, duration ] = __ENV.PROFILE.split(':'); let vus_read = Math.ceil(__ENV.CLIENTS/100*(100-parseInt(write))) let vus_write = __ENV.CLIENTS - vus_read -const payload = crypto.randomBytes(1024*parseInt(__ENV.WRITE_OBJ_SIZE)) +const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE)); let nodes = __ENV.NODES.split(',') // node1.neofs let rand_node = nodes[Math.floor(Math.random()*nodes.length)]; @@ -63,6 +63,7 @@ export const options = { }; export function obj_write() { + const { payload } = generator.genPayload(false); let data = { field: uuidv4(), file: http.file(payload, "random.data"), diff --git a/scenarios/s3.js b/scenarios/s3.js index f0ad09c..0d15f9f 100644 --- a/scenarios/s3.js +++ b/scenarios/s3.js @@ -1,5 +1,5 @@ +import datagen from 'k6/x/neofs/datagen'; import s3 from 'k6/x/neofs/s3'; -import crypto from 'k6/crypto'; import { SharedArray } from 'k6/data'; import { sleep } from 'k6'; @@ -28,7 +28,7 @@ const [ write, duration ] = __ENV.PROFILE.split(':'); let vus_read = Math.ceil(__ENV.CLIENTS/100*(100-parseInt(write))) let vus_write = __ENV.CLIENTS - vus_read -const payload = crypto.randomBytes(1024*parseInt(__ENV.WRITE_OBJ_SIZE)) +const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE)); let nodes = __ENV.NODES.split(',') let rand_node = nodes[Math.floor(Math.random()*nodes.length)]; @@ -79,6 +79,8 @@ export function obj_write() { let bucket = bucket_list[Math.floor(Math.random()*bucket_list.length)]; + + const { payload } = generator.genPayload(false); let resp = s3_cli.put(bucket, key, payload) if (!resp.success) {