From 6d1e7eb49eb28ab2f05ec0aff74b0f7f55e609ed Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 14 Jun 2024 10:41:15 +0300 Subject: [PATCH] [#145] native: Allow to specify max_obj_size For locally prepared objects it is possible now to specify cut size. Signed-off-by: Dmitrii Stepanov --- README.md | 3 ++- examples/native.js | 2 +- examples/native_onsite.js | 2 +- internal/native/client.go | 12 +++++++++--- internal/native/native.go | 17 ++++++++++++++++- scenarios/grpc.js | 4 ++-- scenarios/grpc_car.js | 4 ++-- scenarios/verify.js | 5 ++--- 8 files changed, 35 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 4d9a679..ce3bee3 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,11 @@ Create native client with `connect` method. Arguments: - dial timeout in seconds (0 for the default value) - stream timeout in seconds (0 for the default value) - generate object header on the client side (for big object - split locally too) +- max size for generated object header on the client side (for big object - the size that the object is splitted into) ```js import native from 'k6/x/frostfs/native'; -const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0, false) +const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0, false, 0) ``` ### Methods diff --git a/examples/native.js b/examples/native.js index 2ea5e89..ff3cb40 100644 --- a/examples/native.js +++ b/examples/native.js @@ -3,7 +3,7 @@ import { fail } from "k6"; import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js'; const payload = open('../go.sum', 'b'); -const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb", 0, 0, false) +const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb", 0, 0, false, 0) export const options = { stages: [ diff --git a/examples/native_onsite.js b/examples/native_onsite.js index ce2e24b..d987f34 100644 --- a/examples/native_onsite.js +++ b/examples/native_onsite.js @@ -3,7 +3,7 @@ import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js'; const payload = open('../go.sum', 'b'); const container = "AjSxSNNXbJUDPqqKYm1VbFVDGCakbpUNH8aGjPmGAH3B" -const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0, false) +const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0, false, 0) const frostfs_obj = frostfs_cli.onsite(container, payload) export const options = { diff --git a/internal/native/client.go b/internal/native/client.go index 5979ae3..944bd66 100644 --- a/internal/native/client.go +++ b/internal/native/client.go @@ -35,6 +35,7 @@ type ( tok session.Object cli *client.Client prepareLocally bool + maxObjSize uint64 } PutResponse struct { @@ -71,6 +72,7 @@ type ( hdr object.Object payload []byte prepareLocally bool + maxObjSize uint64 } ) @@ -103,7 +105,7 @@ func (c *Client) Put(containerID string, headers map[string]string, payload data o.SetOwnerID(owner) o.SetAttributes(attrs...) - resp, err := put(c.vu, c.cli, c.prepareLocally, &tok, &o, payload, chunkSize) + resp, err := put(c.vu, c.cli, c.prepareLocally, &tok, &o, payload, chunkSize, c.maxObjSize) if err != nil { return PutResponse{Success: false, Error: err.Error()} } @@ -373,6 +375,7 @@ func (c *Client) Onsite(containerID string, payload datagen.Payload) PreparedObj hdr: *obj, payload: data, prepareLocally: c.prepareLocally, + maxObjSize: c.maxObjSize, } } @@ -398,7 +401,7 @@ func (p PreparedObject) Put(headers map[string]string) PutResponse { return PutResponse{Success: false, Error: err.Error()} } - _, err = put(p.vu, p.cli, p.prepareLocally, nil, &obj, datagen.NewFixedPayload(p.payload), 0) + _, err = put(p.vu, p.cli, p.prepareLocally, nil, &obj, datagen.NewFixedPayload(p.payload), 0, p.maxObjSize) if err != nil { return PutResponse{Success: false, Error: err.Error()} } @@ -413,7 +416,7 @@ func (s epochSource) CurrentEpoch() uint64 { } func put(vu modules.VU, cli *client.Client, prepareLocally bool, tok *session.Object, - hdr *object.Object, payload datagen.Payload, chunkSize int, + hdr *object.Object, payload datagen.Payload, chunkSize int, maxObjSize uint64, ) (*client.ResObjectPut, error) { bufSize := defaultBufferSize if chunkSize > 0 { @@ -441,6 +444,9 @@ func put(vu modules.VU, cli *client.Client, prepareLocally bool, tok *session.Ob prm.MaxSize = res.Info().MaxObjectSize() prm.EpochSource = epochSource(res.Info().CurrentEpoch()) prm.WithoutHomomorphHash = true + if maxObjSize > 0 { + prm.MaxSize = maxObjSize + } } objectWriter, err := cli.ObjectPutInit(vu.Context(), prm) diff --git a/internal/native/native.go b/internal/native/native.go index d3238a1..bb92ed8 100644 --- a/internal/native/native.go +++ b/internal/native/native.go @@ -52,13 +52,17 @@ func (n *Native) Exports() modules.Exports { return modules.Exports{Default: n} } -func (n *Native) Connect(endpoint, hexPrivateKey string, dialTimeout, streamTimeout int, prepareLocally bool) (*Client, error) { +func (n *Native) Connect(endpoint, hexPrivateKey string, dialTimeout, streamTimeout int, prepareLocally bool, maxObjSize int) (*Client, error) { var ( cli client.Client pk *keys.PrivateKey err error ) + if maxObjSize < 0 { + return nil, fmt.Errorf("max object size value must be positive") + } + pk, err = keys.NewPrivateKey() if len(hexPrivateKey) != 0 { pk, err = keys.NewPrivateKeyFromHex(hexPrivateKey) @@ -114,6 +118,16 @@ func (n *Native) Connect(endpoint, hexPrivateKey string, dialTimeout, streamTime tok.SetAuthKey(&key) tok.SetExp(exp) + if prepareLocally && maxObjSize > 0 { + res, err := cli.NetworkInfo(n.vu.Context(), client.PrmNetworkInfo{}) + if err != nil { + return nil, err + } + if uint64(maxObjSize) > res.Info().MaxObjectSize() { + return nil, fmt.Errorf("max object size must be not greater than %d bytes", res.Info().MaxObjectSize()) + } + } + // register metrics objPutSuccess, _ = stats.Registry.NewMetric("frostfs_obj_put_success", metrics.Counter) @@ -140,5 +154,6 @@ func (n *Native) Connect(endpoint, hexPrivateKey string, dialTimeout, streamTime tok: tok, cli: &cli, prepareLocally: prepareLocally, + maxObjSize: uint64(maxObjSize), }, nil } diff --git a/scenarios/grpc.js b/scenarios/grpc.js index 28ef12b..d18b1a0 100644 --- a/scenarios/grpc.js +++ b/scenarios/grpc.js @@ -31,8 +31,8 @@ const grpc_endpoint = const grpc_client = native.connect( grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5, __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60, - __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' - : false); + __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false, + 1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0')); const log = logging.new().withField('endpoint', grpc_endpoint); const registry_enabled = !!__ENV.REGISTRY_FILE; diff --git a/scenarios/grpc_car.js b/scenarios/grpc_car.js index 8ee5d46..296d654 100644 --- a/scenarios/grpc_car.js +++ b/scenarios/grpc_car.js @@ -30,8 +30,8 @@ const grpc_endpoint = const grpc_client = native.connect( grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5, __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60, - __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : - false); + __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false, + 1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0')); const log = logging.new().withField('endpoint', grpc_endpoint); const registry_enabled = !!__ENV.REGISTRY_FILE; diff --git a/scenarios/verify.js b/scenarios/verify.js index e2f0d79..8cdd24f 100644 --- a/scenarios/verify.js +++ b/scenarios/verify.js @@ -44,9 +44,8 @@ if (__ENV.GRPC_ENDPOINTS) { grpc_client = native.connect( grpcEndpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 0, __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 0, - __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : - false, - ''); + __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false, + 1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0')); } // Connect to random S3 endpoint