[#17] Implement k6 extension for random data generation

It improves payload generation in our scenarios. Current implementation
of scenarios generates single random payload at the start and then sends this
same payload on every request. More realistic test is to generate unique payload
for each request. However, this is an expensive operation that can easily cause
a bottleneck on K6 side when we run multiple writing VUs. So instead we generate
a random buffer with some extra bytes and then take slices of this buffer thus
producing a random payload for each request.

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>

.gitignore

@@ -0,0 +1 @@
+k6

internal/datagen/datagen.go

@@ -0,0 +1,42 @@
+package datagen
+
+import (
+	"go.k6.io/k6/js/modules"
+)
+
+// RootModule is the global module object type. It is instantiated once per test
+// run and will be used to create k6/x/neofs/registry module instances for each VU.
+type RootModule struct{}
+
+// Datagen represents an instance of the module for every VU.
+type Datagen struct {
+	vu modules.VU
+}
+
+// Ensure the interfaces are implemented correctly.
+var (
+	_ modules.Instance = &Datagen{}
+	_ modules.Module   = &RootModule{}
+)
+
+func init() {
+	modules.Register("k6/x/neofs/datagen", new(RootModule))
+}
+
+// NewModuleInstance implements the modules.Module interface and returns
+// a new instance for each VU.
+func (r *RootModule) NewModuleInstance(vu modules.VU) modules.Instance {
+	mi := &Datagen{vu: vu}
+	return mi
+}
+
+// Exports implements the modules.Instance interface and returns the exports
+// of the JS module.
+func (d *Datagen) Exports() modules.Exports {
+	return modules.Exports{Default: d}
+}
+
+func (d *Datagen) Generator(size int) *Generator {
+	g := NewGenerator(d.vu, size)
+	return &g
+}

internal/datagen/generator.go

@@ -0,0 +1,77 @@
+package datagen
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"math/rand"
+	"time"
+
+	"github.com/dop251/goja"
+	"go.k6.io/k6/js/modules"
+)
+
+type (
+	// Generator stores buffer of random bytes with some tail and returns data slices with
+	// an increasing offset so that we receive a different set of bytes after each call without
+	// re-generation of the entire buffer from scratch:
+	//
+	//   [<----------size----------><-tail->]
+	//   [<----------slice0-------->........]
+	//   [.<----------slice1-------->.......]
+	//   [..<----------slice2-------->......]
+	Generator struct {
+		vu     modules.VU
+		size   int
+		buf    []byte
+		offset int
+	}
+
+	GenPayloadResponse struct {
+		Payload goja.ArrayBuffer
+		Hash    string
+	}
+)
+
+// TailSize specifies number of extra random bytes in the buffer tail.
+const TailSize = 1024
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+func NewGenerator(vu modules.VU, size int) Generator {
+	return Generator{vu: vu, size: size, buf: nil, offset: 0}
+}
+
+func (g *Generator) GenPayload(calcHash bool) GenPayloadResponse {
+	data := g.nextSlice()
+
+	dataHash := ""
+	if calcHash {
+		hashBytes := sha256.Sum256(data)
+		dataHash = hex.EncodeToString(hashBytes[:])
+	}
+
+	payload := g.vu.Runtime().NewArrayBuffer(data)
+	return GenPayloadResponse{Payload: payload, Hash: dataHash}
+}
+
+func (g *Generator) nextSlice() []byte {
+	if g.buf == nil {
+		// Allocate buffer with extra tail for sliding and populate it with random bytes
+		g.buf = make([]byte, g.size+TailSize)
+		rand.Read(g.buf) // Per docs, err is always nil here
+	}
+
+	result := g.buf[g.offset : g.offset+g.size]
+
+	// Shift the offset for the next call. If we've used our entire tail, then erase
+	// the buffer so that on the next call it is regenerated anew
+	g.offset += 1
+	if g.offset >= TailSize {
+		g.buf = nil
+		g.offset = 0
+	}
+
+	return result
+}

neofs.go

@@ -1,6 +1,7 @@
 package xk6_neofs
 
 import (
+	_ "github.com/nspcc-dev/xk6-neofs/internal/datagen"
 	_ "github.com/nspcc-dev/xk6-neofs/internal/native"
 	_ "github.com/nspcc-dev/xk6-neofs/internal/s3"
 	"go.k6.io/k6/js/modules"

scenarios/grpc.js

@@ -1,5 +1,5 @@
+import datagen from 'k6/x/neofs/datagen';
 import native from 'k6/x/neofs/native';
-import crypto from 'k6/crypto';
 import { SharedArray } from 'k6/data';
 import { sleep } from 'k6';
 
@@ -26,7 +26,7 @@ const [ write, duration ] = __ENV.PROFILE.split(':');
 let vus_read = Math.ceil(__ENV.CLIENTS/100*(100-parseInt(write)))
 let vus_write = __ENV.CLIENTS - vus_read
 
-const payload = crypto.randomBytes(1024*parseInt(__ENV.WRITE_OBJ_SIZE))
+const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE));
 
 let nodes = __ENV.NODES.split(',')
 let rand_node = nodes[Math.floor(Math.random()*nodes.length)];
@@ -72,6 +72,7 @@ export function obj_write() {
     }
     let container = container_list[Math.floor(Math.random()*container_list.length)];
 
+    const { payload } = generator.genPayload(false);
     let resp = neofs_cli.put( container, headers, payload);
     if (!resp.success) {
         console.log(resp.error);

scenarios/http.js

@@ -1,5 +1,5 @@
+import datagen from 'k6/x/neofs/datagen';
 import http from 'k6/http';
-import crypto from 'k6/crypto';
 import { SharedArray } from 'k6/data';
 import { sleep } from 'k6';
 
@@ -24,7 +24,7 @@ const [ write, duration ] = __ENV.PROFILE.split(':');
 let vus_read = Math.ceil(__ENV.CLIENTS/100*(100-parseInt(write)))
 let vus_write = __ENV.CLIENTS - vus_read
 
-const payload = crypto.randomBytes(1024*parseInt(__ENV.WRITE_OBJ_SIZE))
+const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE));
 
 let nodes = __ENV.NODES.split(',') // node1.neofs
 let rand_node = nodes[Math.floor(Math.random()*nodes.length)];
@@ -63,6 +63,7 @@ export const options = {
 };
 
 export function obj_write() {
+    const { payload } = generator.genPayload(false);
     let data = {
         field: uuidv4(),
         file: http.file(payload, "random.data"),

scenarios/s3.js

@@ -1,5 +1,5 @@
+import datagen from 'k6/x/neofs/datagen';
 import s3 from 'k6/x/neofs/s3';
-import crypto from 'k6/crypto';
 import { SharedArray } from 'k6/data';
 import { sleep } from 'k6';
 
@@ -28,7 +28,7 @@ const [ write, duration ] = __ENV.PROFILE.split(':');
 let vus_read = Math.ceil(__ENV.CLIENTS/100*(100-parseInt(write)))
 let vus_write = __ENV.CLIENTS - vus_read
 
-const payload = crypto.randomBytes(1024*parseInt(__ENV.WRITE_OBJ_SIZE))
+const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE));
 
 let nodes = __ENV.NODES.split(',')
 let rand_node = nodes[Math.floor(Math.random()*nodes.length)];
@@ -79,6 +79,8 @@ export function obj_write() {
     
 
     let bucket = bucket_list[Math.floor(Math.random()*bucket_list.length)];
+
+    const { payload } = generator.genPayload(false);
     let resp = s3_cli.put(bucket, key, payload)
     
     if (!resp.success) {