From 025facee96d020218269ae810d61309751761875 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 28 Feb 2023 14:12:12 +0300 Subject: [PATCH] [#13] Allow to use english text in the payload Signed-off-by: Evgenii Stratonikov --- go.mod | 1 + go.sum | 2 ++ internal/datagen/datagen.go | 6 ++-- internal/datagen/generator.go | 55 +++++++++++++++++++++++------- internal/datagen/generator_test.go | 10 +++--- scenarios/grpc.js | 2 +- scenarios/http.js | 2 +- scenarios/run_scenarios.md | 1 + scenarios/s3.js | 2 +- 9 files changed, 58 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index bdcfa70..fef9f83 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.15.5 github.com/aws/aws-sdk-go-v2/service/s3 v1.26.9 github.com/dop251/goja v0.0.0-20230427124612-428fc442ff5f + github.com/go-loremipsum/loremipsum v1.1.3 github.com/google/uuid v1.3.0 github.com/joho/godotenv v1.5.1 github.com/nspcc-dev/neo-go v0.101.1 diff --git a/go.sum b/go.sum index 793746b..2b9897a 100644 --- a/go.sum +++ b/go.sum @@ -161,6 +161,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-loremipsum/loremipsum v1.1.3 h1:ZRhA0ZmJ49lGe5HhWeMONr+iGftWDsHfrYBl5ktDXso= +github.com/go-loremipsum/loremipsum v1.1.3/go.mod h1:OJQjXdvwlG9hsyhmMQoT4HOm4DG4l62CYywebw0XBoo= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sourcemap/sourcemap v2.1.4-0.20211119122758-180fcef48034+incompatible h1:bopx7t9jyUNX1ebhr0G4gtQWmUOgwQRI0QsYhdYLgkU= github.com/go-sourcemap/sourcemap v2.1.4-0.20211119122758-180fcef48034+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= diff --git a/internal/datagen/datagen.go b/internal/datagen/datagen.go index d03ed06..7994875 100644 --- a/internal/datagen/datagen.go +++ b/internal/datagen/datagen.go @@ -1,6 +1,8 @@ package datagen import ( + "strings" + "go.k6.io/k6/js/modules" ) @@ -36,7 +38,7 @@ func (d *Datagen) Exports() modules.Exports { return modules.Exports{Default: d} } -func (d *Datagen) Generator(size int) *Generator { - g := NewGenerator(d.vu, size) +func (d *Datagen) Generator(size int, typ string) *Generator { + g := NewGenerator(d.vu, size, strings.ToLower(typ)) return &g } diff --git a/internal/datagen/generator.go b/internal/datagen/generator.go index e4a2927..c6f1733 100644 --- a/internal/datagen/generator.go +++ b/internal/datagen/generator.go @@ -1,12 +1,13 @@ package datagen import ( + "bytes" "crypto/sha256" "encoding/hex" "math/rand" - "time" "github.com/dop251/goja" + "github.com/go-loremipsum/loremipsum" "go.k6.io/k6/js/modules" ) @@ -24,6 +25,7 @@ type ( size int rand *rand.Rand buf []byte + typ string offset int } @@ -36,20 +38,27 @@ type ( // TailSize specifies number of extra random bytes in the buffer tail. const TailSize = 1024 -func NewGenerator(vu modules.VU, size int) Generator { +var payloadTypes = []string{ + "text", + "random", + "", +} + +func NewGenerator(vu modules.VU, size int, typ string) Generator { if size <= 0 { panic("size should be positive") } - r := rand.New(rand.NewSource(time.Now().UnixNano())) - buf := make([]byte, size+TailSize) - r.Read(buf) - return Generator{ - vu: vu, - size: size, - rand: r, - buf: buf, + var found bool + for i := range payloadTypes { + if payloadTypes[i] == typ { + found = true + } } + if !found { + vu.InitEnv().Logger.Info("Unknown payload type '%s', random will be used.", typ) + } + return Generator{vu: vu, size: size, buf: nil, typ: typ, offset: 0} } func (g *Generator) GenPayload(calcHash bool) GenPayloadResponse { @@ -66,9 +75,24 @@ func (g *Generator) GenPayload(calcHash bool) GenPayloadResponse { } func (g *Generator) nextSlice() []byte { - if g.offset >= TailSize { - g.offset = 0 - g.rand.Read(g.buf) // Per docs, err is always nil here + if g.buf == nil { + // Allocate buffer with extra tail for sliding and populate it with random bytes + g.buf = make([]byte, g.size+TailSize) + rand.Read(g.buf) // Per docs, err is always nil here + switch g.typ { + case "text": + li := loremipsum.New() + b := bytes.NewBuffer(nil) + for b.Len() < g.size+TailSize { + b.WriteString(li.Paragraph()) + b.WriteRune('\n') + } + g.buf = b.Bytes() + default: + // Allocate buffer with extra tail for sliding and populate it with random bytes + g.buf = make([]byte, g.size+TailSize) + rand.Read(g.buf) // Per docs, err is always nil here + } } result := g.buf[g.offset : g.offset+g.size] @@ -76,5 +100,10 @@ func (g *Generator) nextSlice() []byte { // Shift the offset for the next call. If we've used our entire tail, then erase // the buffer so that on the next call it is regenerated anew g.offset += 1 + if g.offset+g.size >= len(g.buf) { + g.buf = nil + g.offset = 0 + } + return result } diff --git a/internal/datagen/generator_test.go b/internal/datagen/generator_test.go index 755c5bb..a82dd25 100644 --- a/internal/datagen/generator_test.go +++ b/internal/datagen/generator_test.go @@ -16,25 +16,25 @@ func TestGenerator(t *testing.T) { t.Run("fails on negative size", func(t *testing.T) { require.Panics(t, func() { - _ = NewGenerator(vu, -1) + _ = NewGenerator(vu, -1, "") }) }) t.Run("fails on zero size", func(t *testing.T) { require.Panics(t, func() { - _ = NewGenerator(vu, 0) + _ = NewGenerator(vu, 0, "") }) }) t.Run("creates slice of specified size", func(t *testing.T) { size := 10 - g := NewGenerator(vu, size) + g := NewGenerator(vu, size, "") slice := g.nextSlice() require.Len(t, slice, size) }) t.Run("creates a different slice on each call", func(t *testing.T) { - g := NewGenerator(vu, 1000) + g := NewGenerator(vu, 1000, "") slice1 := g.nextSlice() slice2 := g.nextSlice() // Each slice should be unique (assuming that 1000 random bytes will never coincide @@ -43,7 +43,7 @@ func TestGenerator(t *testing.T) { }) t.Run("keeps generating slices after consuming entire tail", func(t *testing.T) { - g := NewGenerator(vu, 1000) + g := NewGenerator(vu, 1000, "") initialSlice := g.nextSlice() for i := 0; i < TailSize; i++ { g.nextSlice() diff --git a/scenarios/grpc.js b/scenarios/grpc.js index 5719d5f..8b89443 100644 --- a/scenarios/grpc.js +++ b/scenarios/grpc.js @@ -47,7 +47,7 @@ if (registry_enabled && delete_age) { } -const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE)); +const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE), __ENV.PAYLOAD_TYPE || ""); const scenarios = {}; diff --git a/scenarios/http.js b/scenarios/http.js index 2863fde..0be4e05 100644 --- a/scenarios/http.js +++ b/scenarios/http.js @@ -31,7 +31,7 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und const duration = __ENV.DURATION; -const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE)); +const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE), __ENV.PAYLOAD_TYPE || ""); const scenarios = {}; diff --git a/scenarios/run_scenarios.md b/scenarios/run_scenarios.md index 57a4c37..ba76c79 100644 --- a/scenarios/run_scenarios.md +++ b/scenarios/run_scenarios.md @@ -18,6 +18,7 @@ Scenarios `grpc.js`, `local.js`, `http.js` and `s3.js` support the following opt * `SLEEP_WRITE` - time interval (in seconds) between writing VU iterations. * `SLEEP_READ` - time interval (in seconds) between reading VU iterations. * `SELECTION_SIZE` - size of batch to select for deletion (default: 1000). + * `PAYLOAD_TYPE` - type of an object payload ("random" or "text", default: "random"). Additionally, the profiling extension can be enabled to generate CPU and memory profiles which can be inspected with `go tool pprof file.prof`: ```shell diff --git a/scenarios/s3.js b/scenarios/s3.js index 2ca696c..ae96130 100644 --- a/scenarios/s3.js +++ b/scenarios/s3.js @@ -46,7 +46,7 @@ if (registry_enabled && delete_age) { ); } -const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE)); +const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE), __ENV.PAYLOAD_TYPE || ""); const scenarios = {};