TrueCloudLab/xk6-frostfs
42
1
Fork 20
Code Issues 32 Pull requests 1 Releases Activity Actions

Allow to limit engine size for local scenarios #110

Merged
dstepanov-yadro merged 2 commits from dstepanov-yadro/xk6-frostfs:feat/limit_shard_size into master 2024-09-04 19:51:18 +00:00
Conversation 2 Commits 2 Files changed 12 +47 -30
5 changed files with 47 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit eeededfc18 - Show all commits

20
go.mod
View file

@ -1,11 +1,11 @@
module git.frostfs.info/TrueCloudLab/xk6-frostfs
go 1.19
go 1.20
require (
git.frostfs.info/TrueCloudLab/frostfs-node v0.22.2-0.20230704155826-b520a3049e6f
git.frostfs.info/TrueCloudLab/frostfs-node v0.37.1-0.20231213105742-e39db63827d8
git.frostfs.info/TrueCloudLab/frostfs-s3-gw v0.27.0-rc.2
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230705125206-769f6eec0565
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230928142024-84b9d29fc98c
git.frostfs.info/TrueCloudLab/tzhash v1.8.0
github.com/aws/aws-sdk-go-v2 v1.19.0
github.com/aws/aws-sdk-go-v2/config v1.18.28
@ -15,7 +15,7 @@ require (
github.com/go-loremipsum/loremipsum v1.1.3
github.com/google/uuid v1.3.0
github.com/joho/godotenv v1.5.1
github.com/nspcc-dev/neo-go v0.101.2
github.com/nspcc-dev/neo-go v0.101.5-0.20230808195420-5fc61be5f6c5
github.com/panjf2000/ants/v2 v2.8.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.7.0
@ -27,7 +27,7 @@ require (
)
require (
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb // indirect
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4 // indirect
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 // indirect
git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect
@ -55,13 +55,21 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/dgraph-io/badger/v4 v4.1.0 // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dlclark/regexp2 v1.10.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sourcemap/sourcemap v2.1.4-0.20211119122758-180fcef48034+incompatible // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.1.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v1.12.1 // indirect
github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
@ -90,6 +98,7 @@ require (
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/onsi/gomega v1.20.2 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
@ -103,6 +112,7 @@ require (
github.com/spf13/viper v1.16.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/twmb/murmur3 v1.1.8 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.16.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect

BIN
go.sum
View file

Binary file not shown.

30
internal/local/local.go
View file

@ -19,7 +19,8 @@ import (
metabase "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
writecache "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -127,11 +128,11 @@ func (r *RootModule) GetOrCreateEngine(ctx context.Context, configFile string, c
}
r.ng = engine.New(ngOpts...)
for i, opts := range shardOpts {
if _, err := r.ng.AddShard(opts...); err != nil {
if _, err := r.ng.AddShard(ctx, opts...); err != nil {
return nil, fmt.Errorf("adding shard %d: %v", i, err)
}
}
if err := r.ng.Open(); err != nil {
if err := r.ng.Open(ctx); err != nil {
return nil, fmt.Errorf("opening engine: %v", err)
}
if err := r.ng.Init(ctx); err != nil {
@ -302,15 +303,20 @@ func storageEngineOptionsFromConfig(c *config.Config, debug bool) ([]engine.Opti
opts = append(opts,
shard.WithWriteCache(true),
shard.WithWriteCacheOptions(
writecache.WithPath(wc.Path()),
writecache.WithMaxBatchSize(wc.BoltDB().MaxBatchSize()),
writecache.WithMaxBatchDelay(wc.BoltDB().MaxBatchDelay()),
writecache.WithMaxObjectSize(wc.MaxObjectSize()),
writecache.WithSmallObjectSize(wc.SmallObjectSize()),
writecache.WithFlushWorkersCount(wc.WorkersNumber()),
writecache.WithMaxCacheSize(wc.SizeLimit()),
writecache.WithNoSync(wc.NoSync()),
writecache.WithLogger(&logger.Logger{Logger: log}),
writecache.Options{
Type: writecache.TypeBBolt,
BBoltOptions: []writecachebbolt.Option{
writecachebbolt.WithPath(wc.Path()),
writecachebbolt.WithMaxBatchSize(wc.BoltDB().MaxBatchSize()),
writecachebbolt.WithMaxBatchDelay(wc.BoltDB().MaxBatchDelay()),
writecachebbolt.WithMaxObjectSize(wc.MaxObjectSize()),
writecachebbolt.WithSmallObjectSize(wc.SmallObjectSize()),
writecachebbolt.WithFlushWorkersCount(wc.WorkersNumber()),
writecachebbolt.WithMaxCacheSize(wc.SizeLimit()),
writecachebbolt.WithNoSync(wc.NoSync()),
writecachebbolt.WithLogger(&logger.Logger{Logger: log}),
},
},
),
)
}

25
internal/native/client.go
View file

@ -128,9 +128,9 @@ func (c *Client) Delete(containerID string, objectID string) DeleteResponse {
start := time.Now()
var prm client.PrmObjectDelete
prm.ByID(cliObjectID)
prm.FromContainer(cliContainerID)
prm.WithinSession(tok)
prm.ObjectID = &cliObjectID
prm.ContainerID = &cliContainerID
prm.Session = &tok
_, err = c.cli.ObjectDelete(c.vu.Context(), prm)
if err != nil {
@ -159,11 +159,11 @@ func (c *Client) Get(containerID, objectID string) GetResponse {
start := time.Now()
var prm client.PrmObjectGet
prm.ByID(cliObjectID)
prm.FromContainer(cliContainerID)
prm.WithinSession(tok)
prm.ObjectID = &cliObjectID
prm.ContainerID = &cliContainerID
prm.Session = &tok
var objSize = 0
objSize := 0
err = get(c.cli, prm, c.vu.Context(), func(data []byte) {
objSize += len(data)
})
@ -184,7 +184,7 @@ func get(
ctx context.Context,
onDataChunk func(chunk []byte),
) error {
var buf = make([]byte, defaultBufferSize)
buf := make([]byte, defaultBufferSize)
objectReader, err := cli.ObjectGetInit(ctx, prm)
if err != nil {
@ -227,9 +227,9 @@ func (c *Client) VerifyHash(containerID, objectID, expectedHash string) VerifyHa
}
var prm client.PrmObjectGet
prm.ByID(cliObjectID)
prm.FromContainer(cliContainerID)
prm.WithinSession(tok)
prm.ObjectID = &cliObjectID
prm.ContainerID = &cliContainerID
prm.Session = &tok
hasher := sha256.New()
err = get(c.cli, prm, c.vu.Context(), func(data []byte) {
@ -414,7 +414,8 @@ func (s epochSource) CurrentEpoch() uint64 {
}
func put(vu modules.VU, cli *client.Client, prepareLocally bool, tok *session.Object,
hdr *object.Object, payload []byte, chunkSize int) (*client.ResObjectPut, error) {
hdr *object.Object, payload []byte, chunkSize int,
) (*client.ResObjectPut, error) {
bufSize := defaultBufferSize
if chunkSize > 0 {
bufSize = chunkSize

2
internal/s3local/treeService.go
View file

@ -115,7 +115,7 @@ func (s treeServiceEngineWrapper) GetSubTree(ctx context.Context, bktInfo *data.
return fmt.Errorf("getting children: %v", err)
}
for _, child := range children {
if err := traverse(child, curDepth+1); err != nil {
if err := traverse(child.ID, curDepth+1); err != nil {
return err
}
}