remove http from s3 multipart upload load scenario, protocol would be set in endpoint parameter

Signed-off-by: m.malygina <m.malygina@yadro.com>

.forgejo/workflows/dco.yml

@@ -0,0 +1,21 @@
+name: DCO action
+on: [pull_request]
+
+jobs:
+  dco:
+    name: DCO
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.21'
+
+      - name: Run commit format checker
+        uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2
+        with:
+          from: 'origin/${{ github.event.pull_request.base.ref }}'

.forgejo/workflows/tests.yml

@@ -0,0 +1,38 @@
+name: Tests and linters
+on: [pull_request]
+
+jobs:
+  tests:
+    name: Tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        go_versions: [ '1.20', '1.21' ]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '${{ matrix.go_versions }}'
+          cache: true
+
+      - name: Run tests
+        run: make test
+
+  tests-race:
+    name: Tests with -race
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.21'
+          cache: true
+
+      - name: Run tests
+        run: go test ./... -count=1 -race
+

.github/workflows/dco.yml

@@ -1,21 +0,0 @@
-name: DCO check
-
-on:
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  commits_check_job:
-    runs-on: ubuntu-latest
-    name: Commits Check
-    steps:
-      - name: Get PR Commits
-        id: 'get-pr-commits'
-        uses: tim-actions/get-pr-commits@master
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: DCO Check
-        uses: tim-actions/dco@master
-        with:
-          commits: ${{ steps.get-pr-commits.outputs.commits }}

.github/workflows/go.yml

@@ -1,34 +0,0 @@
-name: Tests
-
-on:
-  pull_request:
-    branches:
-      - master
-    types: [opened, synchronize]
-    paths-ignore:
-      - '**/*.md'
-  workflow_dispatch:
-
-jobs:
-  lint:
-    name: Lint
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest
-          args: --timeout=2m
-
-  tests:
-    name: Tests
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        go_versions: [ '1.17', '1.18', '1.19' ]
-      fail-fast: false
-    steps:
-    - uses: actions/checkout@v3

README.md

@@ -74,12 +74,13 @@ const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0, false)
 
 Create a local client with `connect` method. Arguments:
 - local path to frostfs storage node configuration file
+- local path to frostfs storage node configuration directory
 - hex encoded private key (empty value produces random key)
 - whether to use the debug logger (warning: very verbose)
 
 ```js
 import local from 'k6/x/frostfs/local';
-const local_client = local.connect("/path/to/config.yaml", "", false)
+const local_client = local.connect("/path/to/config.yaml", "/path/to/config/dir", "", false)
 ```
 
 ### Methods
@@ -123,6 +124,7 @@ const s3_cli = s3.connect("https://s3.frostfs.devenv:8080", {'no_verify_ssl': 't
 
 Create local s3 client with `connect` method. Arguments:
 - local path to frostfs storage node configuration file
+- local path to frostfs storage node configuration directory
 - parameter map with the following options:
   * `hex_key`: private key to use as a hexadecimal string. A random one is created if none is provided.
   * `node_position`: position of this node in the node array if loading multiple nodes independently (default: 0).
@@ -135,7 +137,7 @@ Create local s3 client with `connect` method. Arguments:
 import local from 'k6/x/frostfs/local';
 const params = {'node_position': 1, 'node_count': 3}
 const bucketMapping = {'mytestbucket': 'GBQDDUM1hdodXmiRHV57EUkFWJzuntsG8BG15wFSwam6'}
-const local_client = local.connect("/path/to/config.yaml", params, bucketMapping)
+const local_client = local.connect("/path/to/config.yaml", "/path/to/config/dir", params, bucketMapping)
 ```
 
 ### Methods

examples/local.js

@@ -2,7 +2,7 @@ import local from 'k6/x/frostfs/local';
 import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js';
 
 const payload = open('../go.sum', 'b');
-const local_cli = local.connect("/path/to/config.yaml", "", false)
+const local_cli = local.connect("/path/to/config.yaml", "/path/to/config/dir", "", false)
 
 export const options = {
     stages: [

examples/s3local.js

@@ -3,7 +3,7 @@ import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js';
 
 const bucket = "testbucket"
 const payload = open('../go.sum', 'b');
-const s3local_cli = s3local.connect("path/to/storage/config.yml", {}, {
+const s3local_cli = s3local.connect("path/to/storage/config.yml", "path/to/storage/config/dir", {}, {
     'testbucket': 'GBQDDUM1hdodXmiRHV57EUkFWJzuntsG8BG15wFSwam6',
 });
 

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
 	go.etcd.io/bbolt v1.3.7
-	go.k6.io/k6 v0.45.0
+	go.k6.io/k6 v0.45.1
 	go.uber.org/zap v1.24.0
 	golang.org/x/sys v0.10.0
 )

go.sum

@@ -379,8 +379,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
 go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
-go.k6.io/k6 v0.45.0 h1:cXFQi0GuHCl0YcI2bl5NWne6J2qRU23E/R/YoqDmcD8=
-go.k6.io/k6 v0.45.0/go.mod h1:8RMnULVviBhE6vbrnNoc9yq9mXvniXrJaLEP0OqGki0=
+go.k6.io/k6 v0.45.1 h1:z+iVxE7Qze2Ka8tKvnjerOsoTuQb8e27Vqd1wcG2IFI=
+go.k6.io/k6 v0.45.1/go.mod h1:SBO/sqx6h/a0lJqEioMEpneb6zULogIyDmz+ufFqtIE=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

internal/local/local.go

@@ -40,6 +40,8 @@ type RootModule struct {
 	mu sync.Mutex
 	// configFile is the name of the configuration file used during one test.
 	configFile string
+	// configDir is the name of the configuration directory used during one test.
+	configDir string
 	// ng is the engine instance used during one test, corresponding to the configFile. Each VU
 	// gets the same engine instance.
 	ng *engine.StorageEngine
@@ -48,7 +50,7 @@ type RootModule struct {
 // Local represents an instance of the module for every VU.
 type Local struct {
 	vu            modules.VU
-	ResolveEngine func(context.Context, string, bool) (*engine.StorageEngine, error)
+	ResolveEngine func(context.Context, string, string, bool) (*engine.StorageEngine, error)
 }
 
 // Ensure the interfaces are implemented correctly.
@@ -71,7 +73,7 @@ func (r *RootModule) NewModuleInstance(vu modules.VU) modules.Instance {
 	return NewLocalModuleInstance(vu, r.GetOrCreateEngine)
 }
 
-func NewLocalModuleInstance(vu modules.VU, resolveEngine func(context.Context, string, bool) (*engine.StorageEngine, error)) *Local {
+func NewLocalModuleInstance(vu modules.VU, resolveEngine func(context.Context, string, string, bool) (*engine.StorageEngine, error)) *Local {
 	return &Local{
 		vu:            vu,
 		ResolveEngine: resolveEngine,
@@ -100,21 +102,22 @@ func checkResourceLimits() error {
 	return nil
 }
 
-// GetOrCreateEngine returns the current engine instance for the given configuration file,
+// GetOrCreateEngine returns the current engine instance for the given configuration file or directory,
 // creating a new one if none exists. Note that the identity of configuration files is their
 // file name for the purposes of test runs.
-func (r *RootModule) GetOrCreateEngine(ctx context.Context, configFile string, debug bool) (*engine.StorageEngine, error) {
+func (r *RootModule) GetOrCreateEngine(ctx context.Context, configFile string, configDir string, debug bool) (*engine.StorageEngine, error) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
-	if len(configFile) == 0 {
-		return nil, errors.New("configFile cannot be empty")
+	if len(configFile) == 0 && len(configDir) == 0 {
+		return nil, errors.New("provide configFile or configDir")
 	}
 
 	// Create and initialize engine for the given configFile if it doesn't exist already
 	if r.ng == nil {
 		r.configFile = configFile
-		appCfg := config.New(configFile, "", "")
+		r.configDir = configDir
+		appCfg := config.New(configFile, configDir, "")
 		ngOpts, shardOpts, err := storageEngineOptionsFromConfig(appCfg, debug)
 		if err != nil {
 			return nil, fmt.Errorf("creating engine options from config: %v", err)
@@ -135,7 +138,11 @@ func (r *RootModule) GetOrCreateEngine(ctx context.Context, configFile string, d
 			return nil, fmt.Errorf("initializing engine: %v", err)
 		}
 	} else if configFile != r.configFile {
-		return nil, fmt.Errorf("GetOrCreateEngine called with mismatching configFile after engine was initialized: got %q, want %q", configFile, r.configFile)
+		return nil, fmt.Errorf("GetOrCreateEngine called with mismatching configFile after engine was "+
+			"initialized: got %q, want %q", configFile, r.configFile)
+	} else if configDir != r.configDir {
+		return nil, fmt.Errorf("GetOrCreateEngine called with mismatching configDir after engine was "+
+			"initialized: got %q, want %q", configDir, r.configDir)
 	}
 
 	return r.ng, nil
@@ -149,10 +156,10 @@ func (s *Local) Exports() modules.Exports {
 
 func (s *Local) VU() modules.VU { return s.vu }
 
-func (s *Local) Connect(configFile, hexKey string, debug bool) (*Client, error) {
-	ng, err := s.ResolveEngine(s.VU().Context(), configFile, debug)
+func (s *Local) Connect(configFile, configDir, hexKey string, debug bool) (*Client, error) {
+	ng, err := s.ResolveEngine(s.VU().Context(), configFile, configDir, debug)
 	if err != nil {
-		return nil, fmt.Errorf("connecting to engine for config %q: %v", configFile, err)
+		return nil, fmt.Errorf("connecting to engine for config - file %q dir %q: %v", configFile, configDir, err)
 	}
 
 	key, err := ParseOrCreateKey(hexKey)

internal/s3local/local.go

@@ -56,7 +56,7 @@ func (s *Local) Exports() modules.Exports {
 	return modules.Exports{Default: s}
 }
 
-func (s *Local) Connect(configFile string, params map[string]string, bucketMapping map[string]string) (*Client, error) {
+func (s *Local) Connect(configFile string, configDir string, params map[string]string, bucketMapping map[string]string) (*Client, error) {
 	// Parse configuration flags.
 	fs := flag.NewFlagSet("s3local", flag.ContinueOnError)
 
@@ -107,9 +107,9 @@ func (s *Local) Connect(configFile string, params map[string]string, bucketMappi
 	objGetDuration, _ = registry.NewMetric("s3local_obj_get_duration", metrics.Trend, metrics.Time)
 
 	// Create S3 layer backed by local storage engine and tree service.
-	ng, err := s.l.ResolveEngine(s.l.VU().Context(), configFile, *debugLogger)
+	ng, err := s.l.ResolveEngine(s.l.VU().Context(), configFile, configDir, *debugLogger)
 	if err != nil {
-		return nil, fmt.Errorf("connecting to engine for config %q: %v", configFile, err)
+		return nil, fmt.Errorf("connecting to engine for config - file %q dir %q: %v", configFile, configDir, err)
 	}
 
 	treeSvc := tree.NewTree(treeServiceEngineWrapper{

scenarios/grpc.js

@@ -49,6 +49,7 @@ if (registry_enabled && delete_age) {
     );
 }
 
+const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
 let obj_to_read_selector = undefined;
 if (registry_enabled) {
     obj_to_read_selector = registry.getSelector(
@@ -57,6 +58,7 @@ if (registry_enabled) {
         __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
         {
             status: "created",
+            age: read_age,
         }
     )
 }
@@ -119,12 +121,17 @@ export function setup() {
     console.log(`Writing VUs:                   ${write_vu_count}`);
     console.log(`Deleting VUs:                  ${delete_vu_count}`);
     console.log(`Total VUs:                     ${total_vu_count}`);
+    
+    const start_timestamp = Date.now()
+    console.log(`Load started at:               ${Date(start_timestamp).toString()}`)
 }
 
 export function teardown(data) {
     if (obj_registry) {
         obj_registry.close();
     }
+    const end_timestamp = Date.now()
+    console.log(`Load finished at:              ${Date(end_timestamp).toString()}`)
 }
 
 export function handleSummary(data) {

scenarios/grpc_car.js

@@ -49,6 +49,7 @@ if (registry_enabled && delete_age) {
     );
 }
 
+const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
 let obj_to_read_selector = undefined;
 if (registry_enabled) {
     obj_to_read_selector = registry.getSelector(
@@ -57,6 +58,7 @@ if (registry_enabled) {
         __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
         {
             status: "created",
+            age: read_age,
         }
     )
 }
@@ -143,12 +145,17 @@ export function setup() {
     console.log(`Read rate:                     ${read_rate}`);
     console.log(`Writing rate:                  ${write_rate}`);
     console.log(`Delete rate:                   ${delete_rate}`);
+    
+    const start_timestamp = Date.now()
+    console.log(`Load started at:               ${Date(start_timestamp).toString()}`)
 }
 
 export function teardown(data) {
     if (obj_registry) {
         obj_registry.close();
     }
+    const end_timestamp = Date.now()
+    console.log(`Load finished at:              ${Date(end_timestamp).toString()}`)
 }
 
 export function handleSummary(data) {

scenarios/http.js

@@ -71,12 +71,17 @@ export function setup() {
     console.log(`Reading VUs:                   ${read_vu_count}`);
     console.log(`Writing VUs:                   ${write_vu_count}`);
     console.log(`Total VUs:                     ${total_vu_count}`);
+
+    const start_timestamp = Date.now()
+    console.log(`Load started at:               ${Date(start_timestamp).toString()}`)
 }
 
 export function teardown(data) {
     if (obj_registry) {
         obj_registry.close();
     }
+    const end_timestamp = Date.now()
+    console.log(`Load finished at:              ${Date(end_timestamp).toString()}`)
 }
 
 export function handleSummary(data) {

scenarios/local.js

@@ -21,9 +21,10 @@ const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
 const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
 
 const config_file = __ENV.CONFIG_FILE;
+const config_dir = __ENV.CONFIG_DIR;
 const debug_logger = (__ENV.DEBUG_LOGGER || 'false') == 'true';
-const local_client = local.connect(config_file, '', debug_logger);
-const log = logging.new().withField("config", config_file);
+const local_client = local.connect(config_file, config_dir, '', debug_logger);
+const log = logging.new().withFields({"config_file": config_file,"config_dir": config_dir});
 
 const registry_enabled = !!__ENV.REGISTRY_FILE;
 const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined;
@@ -100,12 +101,17 @@ export function setup() {
     console.log(`Writing VUs:                   ${write_vu_count}`);
     console.log(`Deleting VUs:                  ${delete_vu_count}`);
     console.log(`Total VUs:                     ${total_vu_count}`);
+
+    const start_timestamp = Date.now()
+    console.log(`Load started at:               ${Date(start_timestamp).toString()}`)
 }
 
 export function teardown(data) {
     if (obj_registry) {
         obj_registry.close();
     }
+    const end_timestamp = Date.now()
+    console.log(`Load finished at:              ${Date(end_timestamp).toString()}`)
 }
 
 export function handleSummary(data) {

scenarios/preset/helpers/aws_cli.py

@@ -1,6 +1,6 @@
 import uuid
 
-from helpers.cmd import execute_cmd
+from helpers.cmd import execute_cmd, log
 
 
 def create_bucket(endpoint, versioning, location, no_verify_ssl):
@@ -13,22 +13,24 @@ def create_bucket(endpoint, versioning, location, no_verify_ssl):
     cmd_line_ver = f"aws {no_verify_ssl_str} s3api put-bucket-versioning --bucket {bucket_name} " \
                    f"--versioning-configuration Status=Enabled --endpoint {endpoint} "
 
-    out, success = execute_cmd(cmd_line)
+    output, success = execute_cmd(cmd_line)
 
-    if not success and "succeeded and you already own it" not in out:
-        print(f" > Bucket {bucket_name} has not been created:\n{out}")
+    if not success and "succeeded and you already own it" not in output:
+        log(f"{cmd_line}\n"
+            f"Bucket {bucket_name} has not been created:\n"
+            f"Error: {output}", endpoint)
         return False
-    
-    print(f"cmd: {cmd_line}")
 
     if versioning == "True":
-        out, success = execute_cmd(cmd_line_ver)
+        output, success = execute_cmd(cmd_line_ver)
         if not success:
-            print(f" > Bucket versioning has not been applied for bucket {bucket_name}:\n{out}")
+            log(f"{cmd_line_ver}\n"
+                f"Bucket versioning has not been applied for bucket {bucket_name}\n"
+                f"Error: {output}", endpoint)
         else:
-            print(f" > Bucket versioning has been applied.")
+            log(f"Bucket versioning has been applied for bucket {bucket_name}", endpoint)
     
-    print(f"Created bucket: {bucket_name} via endpoint {endpoint}")
+    log(f"Created bucket: {bucket_name}", endpoint)
     return bucket_name
 
 
@@ -37,10 +39,12 @@ def upload_object(bucket, payload_filepath, endpoint, no_verify_ssl):
     no_verify_ssl_str = "--no-verify-ssl" if no_verify_ssl else ""
     cmd_line = f"aws {no_verify_ssl_str} s3api put-object --bucket {bucket} --key {object_name} " \
                f"--body {payload_filepath} --endpoint {endpoint}"
-    out, success = execute_cmd(cmd_line)
+    output, success = execute_cmd(cmd_line)
 
     if not success:
-        print(f" > Object {object_name} has not been uploaded.")
+        log(f"{cmd_line}\n"
+            f"Object {object_name} has not been uploaded\n"
+            f"Error: {output}", endpoint)
         return False
     
     return bucket, endpoint, object_name

scenarios/preset/helpers/cmd.py

@@ -1,9 +1,12 @@
 import os
 import shlex
 import sys
-
+from datetime import datetime
 from subprocess import check_output, CalledProcessError, STDOUT
 
+def log(message, endpoint):
+    time = datetime.utcnow()
+    print(f"{time} at {endpoint}: {message}")
 
 def execute_cmd(cmd_line):
     cmd_args = shlex.split(cmd_line)

scenarios/preset/helpers/frostfs_cli.py

@@ -1,47 +1,61 @@
 import re
-
-from helpers.cmd import execute_cmd
-
+from helpers.cmd import execute_cmd, log
 
 def create_container(endpoint, policy, wallet_file, wallet_config):
-    cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} container create --wallet {wallet_file} --config {wallet_config} " \
+    if wallet_file:
+        wallet_file = "--wallet " + wallet_file
+    if wallet_config:
+        wallet_config = "--config " + wallet_config
+    cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} container create {wallet_file} {wallet_config} " \
                f" --policy '{policy}' --basic-acl public-read-write --await"
 
     output, success = execute_cmd(cmd_line)
 
     if not success:
-        print(f" > Container has not been created:\n{output}")
+        log(f"{cmd_line}\n"
+            f"Container has not been created\n"
+            f"{output}", endpoint)
         return False
     
     try:
         fst_str = output.split('\n')[0]
     except Exception:
-        print(f"Got empty output: {output}")
+        log(f"{cmd_line}\n"
+            f"Incorrect output\n"
+            f"Output: {output or '<empty>'}", endpoint)
         return False
     splitted = fst_str.split(": ")
     if len(splitted) != 2:
-        raise ValueError(f"no CID was parsed from command output: \t{fst_str}")
+        raise ValueError(f"no CID was parsed from command output:\t{fst_str}")
 
-    print(f"Created container: {splitted[1]} via endpoint {endpoint}")
+    log(f"Created container {splitted[1]}", endpoint)
 
     return splitted[1]
 
 
 def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_config):
     object_name = ""
-    cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} object put --file {payload_filepath} --wallet {wallet_file} --config {wallet_config} " \
+    if wallet_file:
+        wallet_file = "--wallet " + wallet_file
+    if wallet_config:
+        wallet_config = "--config " + wallet_config
+    cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} object put --file {payload_filepath} {wallet_file} {wallet_config} " \
                f"--cid {container} --no-progress"
     output, success = execute_cmd(cmd_line)
 
     if not success:
-        print(f" > Object {object_name} has not been uploaded:\n{output}")
+        log(f"{cmd_line}\n"
+            f"Object {object_name} has not been uploaded\n"
+            f"Error: {output}", endpoint)
         return False
 
     try:
         # taking second string from command output
         snd_str = output.split('\n')[1]
     except Exception:
-        print(f"Got empty input: {output}")
+        log(f"{cmd_line}\n"
+            f"Incorrect output\n"
+            f"Output: {output or '<empty>'}", endpoint)
         return False
     splitted = snd_str.split(": ")
     if len(splitted) != 2:
@@ -50,32 +64,42 @@ def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_con
 
 
 def get_object(cid, oid, endpoint, out_filepath, wallet_file, wallet_config):
-    cmd_line = f"frostfs-cli object get -r {endpoint} --cid {cid} --oid {oid} --wallet {wallet_file} --config {wallet_config} " \
+    if wallet_file:
+        wallet_file = "--wallet " + wallet_file
+    if wallet_config:
+        wallet_config = "--config " + wallet_config
+    cmd_line = f"frostfs-cli object get -r {endpoint} --cid {cid} --oid {oid} {wallet_file} {wallet_config} " \
                f"--file {out_filepath}"
 
     output, success = execute_cmd(cmd_line)
 
     if not success:
-        print(f" > Failed to get object {output} from container {cid} \r\n"
-              f" > Error: {output}")
+        log(f"{cmd_line}\n"
+            f"Failed to get object {oid} from container {cid}\n"
+            f"Error: {output}", endpoint)
         return False
 
     return True
 
 
 def search_object_by_id(cid, oid, endpoint, wallet_file, wallet_config, ttl=2):
-    cmd_line = f"frostfs-cli object search --ttl {ttl} -r {endpoint} --cid {cid} --oid {oid} --wallet {wallet_file} --config {wallet_config} "
+    if wallet_file:
+        wallet_file = "--wallet " + wallet_file
+    if wallet_config:
+        wallet_config = "--config " + wallet_config
+    cmd_line = f"frostfs-cli object search --ttl {ttl} -r {endpoint} --cid {cid} --oid {oid} {wallet_file} {wallet_config} "
 
     output, success = execute_cmd(cmd_line)
 
     if not success:
-        print(f" > Failed to search object {oid} for container {cid} \r\n"
-              f" > Error: {output}")
+        log(f"{cmd_line}\n"
+            f"Failed to search object {oid} for container {cid}\n"
+            f"Error: {output}", endpoint)
         return False
 
     re_rst = re.search(r'Found (\d+) objects', output)
 
     if not re_rst:
-        raise Exception("Failed to parce search results")
+        raise Exception("Failed to parse search results")
 
     return re_rst.group(1)

scenarios/preset/preset_grpc.py

@@ -3,7 +3,6 @@
 import argparse
 from itertools import cycle
 import json
-import random
 import sys
 import tempfile
 import time
@@ -33,7 +32,7 @@ parser.add_argument('--endpoint', help='Nodes addresses separated by comma.')
 parser.add_argument('--update', help='Save existed containers')
 parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
 parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Default = 50', default=50)
-parser.add_argument('--sleep', help='Time to sleep between container creation and object PUT (in seconds), '
+parser.add_argument('--sleep', help='Time to sleep between containers creation and objects upload (in seconds), '
                                     'Default = 8', default=8)
 
 args: Namespace = parser.parse_args()

scenarios/preset/preset_s3.py

@@ -25,7 +25,7 @@ parser.add_argument('--versioning', help='True/False, False by default.')
 parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
 parser.add_argument('--no-verify-ssl', help='Ignore SSL verifications', action='store_true')
 parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Default = 50', default=50)
-parser.add_argument('--sleep', help='Time to sleep between container creation and object PUT (in seconds), '
+parser.add_argument('--sleep', help='Time to sleep between buckets creation and objects upload (in seconds), '
                                     'Default = 8', default=8)
 
 args = parser.parse_args()

scenarios/s3.js

@@ -48,6 +48,7 @@ if (registry_enabled && delete_age) {
     );
 }
 
+const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
 let obj_to_read_selector = undefined;
 if (registry_enabled) {
     obj_to_read_selector = registry.getSelector(
@@ -56,6 +57,7 @@ if (registry_enabled) {
         __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
         {
             status: "created",
+            age: read_age,
         }
     )
 }
@@ -116,12 +118,17 @@ export function setup() {
     console.log(`Writing VUs:                   ${write_vu_count}`);
     console.log(`Deleting VUs:                  ${delete_vu_count}`);
     console.log(`Total VUs:                     ${total_vu_count}`);
+    
+    const start_timestamp = Date.now()
+    console.log(`Load started at:               ${Date(start_timestamp).toString()}`)
 }
 
 export function teardown(data) {
     if (obj_registry) {
         obj_registry.close();
     }
+    const end_timestamp = Date.now()
+    console.log(`Load finished at:              ${Date(end_timestamp).toString()}`)
 }
 
 export function handleSummary(data) {

scenarios/s3_car.js

@@ -48,6 +48,7 @@ if (registry_enabled && delete_age) {
     );
 }
 
+const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
 let obj_to_read_selector = undefined;
 if (registry_enabled) {
     obj_to_read_selector = registry.getSelector(
@@ -56,6 +57,7 @@ if (registry_enabled) {
         __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
         {
             status: "created",
+            age: read_age,
         }
     )
 }
@@ -143,12 +145,17 @@ export function setup() {
     console.log(`Read rate:                     ${read_rate}`);
     console.log(`Writing rate:                  ${write_rate}`);
     console.log(`Delete rate:                   ${delete_rate}`);
+
+    const start_timestamp = Date.now()
+    console.log(`Load started at:               ${Date(start_timestamp).toString()}`)
 }
 
 export function teardown(data) {
     if (obj_registry) {
         obj_registry.close();
     }
+    const end_timestamp = Date.now()
+    console.log(`Load finished at:              ${Date(end_timestamp).toString()}`)
 }
 
 export function handleSummary(data) {

scenarios/s3_multipart.js

@@ -19,7 +19,9 @@ const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
 // Select random S3 endpoint for current VU
 const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
 const s3_endpoint = s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)];
-const s3_client = s3.connect(`http://${s3_endpoint}`);
+const no_verify_ssl = __ENV.NO_VERIFY_SSL || "true";
+const connection_args = {no_verify_ssl: no_verify_ssl}
+const s3_client = s3.connect(s3_endpoint, connection_args);
 const log = logging.new().withField("endpoint", s3_endpoint);
 
 const registry_enabled = !!__ENV.REGISTRY_FILE;

scenarios/s3local.js

@@ -36,10 +36,11 @@ const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
 const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
 
 const config_file = __ENV.CONFIG_FILE;
-const s3_client = s3local.connect(config_file, {
+const config_dir = __ENV.CONFIG_DIR;
+const s3_client = s3local.connect(config_file, config_dir, {
     'debug_logger': __ENV.DEBUG_LOGGER || 'false',
 }, bucket_mapping());
-const log = logging.new().withField("config", config_file);
+const log = logging.new().withFields({"config_file": config_file,"config_dir": config_dir});
 
 const registry_enabled = !!__ENV.REGISTRY_FILE;
 const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined;
@@ -98,12 +99,17 @@ export function setup() {
     console.log(`Reading VUs:                   ${read_vu_count}`);
     console.log(`Writing VUs:                   ${write_vu_count}`);
     console.log(`Total VUs:                     ${total_vu_count}`);
+
+    const start_timestamp = Date.now()
+    console.log(`Load started at:               ${Date(start_timestamp).toString()}`)
 }
 
 export function teardown(data) {
     if (obj_registry) {
         obj_registry.close();
     }
+    const end_timestamp = Date.now()
+    console.log(`Load finished at:              ${Date(end_timestamp).toString()}`)
 }
 
 export function handleSummary(data) {