forked from TrueCloudLab/xk6-frostfs
Compare commits
No commits in common. "48a95bc50b84ce4f57b4f4377327a1cdbf06cf62" and "4ef3795e0419565c441b3d15972fe1a3024c9d7c" have entirely different histories.
48a95bc50b
...
4ef3795e04
29 changed files with 115 additions and 205 deletions
|
@ -1,21 +0,0 @@
|
|||
name: DCO action
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
dco:
|
||||
name: DCO
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.21'
|
||||
|
||||
- name: Run commit format checker
|
||||
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2
|
||||
with:
|
||||
from: 'origin/${{ github.event.pull_request.base.ref }}'
|
|
@ -1,38 +0,0 @@
|
|||
name: Tests and linters
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
name: Tests
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
go_versions: [ '1.20', '1.21' ]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '${{ matrix.go_versions }}'
|
||||
cache: true
|
||||
|
||||
- name: Run tests
|
||||
run: make test
|
||||
|
||||
tests-race:
|
||||
name: Tests with -race
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.21'
|
||||
cache: true
|
||||
|
||||
- name: Run tests
|
||||
run: go test ./... -count=1 -race
|
||||
|
0
.forgejo/CODEOWNERS → .github/CODEOWNERS
vendored
0
.forgejo/CODEOWNERS → .github/CODEOWNERS
vendored
0
.forgejo/logo.svg → .github/logo.svg
vendored
0
.forgejo/logo.svg → .github/logo.svg
vendored
Before Width: | Height: | Size: 5.5 KiB After Width: | Height: | Size: 5.5 KiB |
21
.github/workflows/dco.yml
vendored
Normal file
21
.github/workflows/dco.yml
vendored
Normal file
|
@ -0,0 +1,21 @@
|
|||
name: DCO check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
commits_check_job:
|
||||
runs-on: ubuntu-latest
|
||||
name: Commits Check
|
||||
steps:
|
||||
- name: Get PR Commits
|
||||
id: 'get-pr-commits'
|
||||
uses: tim-actions/get-pr-commits@master
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: DCO Check
|
||||
uses: tim-actions/dco@master
|
||||
with:
|
||||
commits: ${{ steps.get-pr-commits.outputs.commits }}
|
34
.github/workflows/go.yml
vendored
Normal file
34
.github/workflows/go.yml
vendored
Normal file
|
@ -0,0 +1,34 @@
|
|||
name: Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
types: [opened, synchronize]
|
||||
paths-ignore:
|
||||
- '**/*.md'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
with:
|
||||
version: latest
|
||||
args: --timeout=2m
|
||||
|
||||
tests:
|
||||
name: Tests
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
go_versions: [ '1.17', '1.18', '1.19' ]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
|
@ -74,13 +74,12 @@ const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0, false)
|
|||
|
||||
Create a local client with `connect` method. Arguments:
|
||||
- local path to frostfs storage node configuration file
|
||||
- local path to frostfs storage node configuration directory
|
||||
- hex encoded private key (empty value produces random key)
|
||||
- whether to use the debug logger (warning: very verbose)
|
||||
|
||||
```js
|
||||
import local from 'k6/x/frostfs/local';
|
||||
const local_client = local.connect("/path/to/config.yaml", "/path/to/config/dir", "", false)
|
||||
const local_client = local.connect("/path/to/config.yaml", "", false)
|
||||
```
|
||||
|
||||
### Methods
|
||||
|
@ -124,7 +123,6 @@ const s3_cli = s3.connect("https://s3.frostfs.devenv:8080", {'no_verify_ssl': 't
|
|||
|
||||
Create local s3 client with `connect` method. Arguments:
|
||||
- local path to frostfs storage node configuration file
|
||||
- local path to frostfs storage node configuration directory
|
||||
- parameter map with the following options:
|
||||
* `hex_key`: private key to use as a hexadecimal string. A random one is created if none is provided.
|
||||
* `node_position`: position of this node in the node array if loading multiple nodes independently (default: 0).
|
||||
|
@ -137,7 +135,7 @@ Create local s3 client with `connect` method. Arguments:
|
|||
import local from 'k6/x/frostfs/local';
|
||||
const params = {'node_position': 1, 'node_count': 3}
|
||||
const bucketMapping = {'mytestbucket': 'GBQDDUM1hdodXmiRHV57EUkFWJzuntsG8BG15wFSwam6'}
|
||||
const local_client = local.connect("/path/to/config.yaml", "/path/to/config/dir", params, bucketMapping)
|
||||
const local_client = local.connect("/path/to/config.yaml", params, bucketMapping)
|
||||
```
|
||||
|
||||
### Methods
|
||||
|
|
|
@ -2,7 +2,7 @@ import local from 'k6/x/frostfs/local';
|
|||
import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js';
|
||||
|
||||
const payload = open('../go.sum', 'b');
|
||||
const local_cli = local.connect("/path/to/config.yaml", "/path/to/config/dir", "", false)
|
||||
const local_cli = local.connect("/path/to/config.yaml", "", false)
|
||||
|
||||
export const options = {
|
||||
stages: [
|
||||
|
|
|
@ -3,7 +3,7 @@ import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js';
|
|||
|
||||
const bucket = "testbucket"
|
||||
const payload = open('../go.sum', 'b');
|
||||
const s3local_cli = s3local.connect("path/to/storage/config.yml", "path/to/storage/config/dir", {}, {
|
||||
const s3local_cli = s3local.connect("path/to/storage/config.yml", {}, {
|
||||
'testbucket': 'GBQDDUM1hdodXmiRHV57EUkFWJzuntsG8BG15wFSwam6',
|
||||
});
|
||||
|
||||
|
|
2
go.mod
2
go.mod
|
@ -21,7 +21,7 @@ require (
|
|||
github.com/spf13/cobra v1.7.0
|
||||
github.com/stretchr/testify v1.8.4
|
||||
go.etcd.io/bbolt v1.3.7
|
||||
go.k6.io/k6 v0.45.1
|
||||
go.k6.io/k6 v0.45.0
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/sys v0.10.0
|
||||
)
|
||||
|
|
4
go.sum
4
go.sum
|
@ -379,8 +379,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
|
|||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
|
||||
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
|
||||
go.k6.io/k6 v0.45.1 h1:z+iVxE7Qze2Ka8tKvnjerOsoTuQb8e27Vqd1wcG2IFI=
|
||||
go.k6.io/k6 v0.45.1/go.mod h1:SBO/sqx6h/a0lJqEioMEpneb6zULogIyDmz+ufFqtIE=
|
||||
go.k6.io/k6 v0.45.0 h1:cXFQi0GuHCl0YcI2bl5NWne6J2qRU23E/R/YoqDmcD8=
|
||||
go.k6.io/k6 v0.45.0/go.mod h1:8RMnULVviBhE6vbrnNoc9yq9mXvniXrJaLEP0OqGki0=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
|
|
|
@ -40,8 +40,6 @@ type RootModule struct {
|
|||
mu sync.Mutex
|
||||
// configFile is the name of the configuration file used during one test.
|
||||
configFile string
|
||||
// configDir is the name of the configuration directory used during one test.
|
||||
configDir string
|
||||
// ng is the engine instance used during one test, corresponding to the configFile. Each VU
|
||||
// gets the same engine instance.
|
||||
ng *engine.StorageEngine
|
||||
|
@ -50,7 +48,7 @@ type RootModule struct {
|
|||
// Local represents an instance of the module for every VU.
|
||||
type Local struct {
|
||||
vu modules.VU
|
||||
ResolveEngine func(context.Context, string, string, bool) (*engine.StorageEngine, error)
|
||||
ResolveEngine func(context.Context, string, bool) (*engine.StorageEngine, error)
|
||||
}
|
||||
|
||||
// Ensure the interfaces are implemented correctly.
|
||||
|
@ -73,7 +71,7 @@ func (r *RootModule) NewModuleInstance(vu modules.VU) modules.Instance {
|
|||
return NewLocalModuleInstance(vu, r.GetOrCreateEngine)
|
||||
}
|
||||
|
||||
func NewLocalModuleInstance(vu modules.VU, resolveEngine func(context.Context, string, string, bool) (*engine.StorageEngine, error)) *Local {
|
||||
func NewLocalModuleInstance(vu modules.VU, resolveEngine func(context.Context, string, bool) (*engine.StorageEngine, error)) *Local {
|
||||
return &Local{
|
||||
vu: vu,
|
||||
ResolveEngine: resolveEngine,
|
||||
|
@ -102,22 +100,21 @@ func checkResourceLimits() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// GetOrCreateEngine returns the current engine instance for the given configuration file or directory,
|
||||
// GetOrCreateEngine returns the current engine instance for the given configuration file,
|
||||
// creating a new one if none exists. Note that the identity of configuration files is their
|
||||
// file name for the purposes of test runs.
|
||||
func (r *RootModule) GetOrCreateEngine(ctx context.Context, configFile string, configDir string, debug bool) (*engine.StorageEngine, error) {
|
||||
func (r *RootModule) GetOrCreateEngine(ctx context.Context, configFile string, debug bool) (*engine.StorageEngine, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
if len(configFile) == 0 && len(configDir) == 0 {
|
||||
return nil, errors.New("provide configFile or configDir")
|
||||
if len(configFile) == 0 {
|
||||
return nil, errors.New("configFile cannot be empty")
|
||||
}
|
||||
|
||||
// Create and initialize engine for the given configFile if it doesn't exist already
|
||||
if r.ng == nil {
|
||||
r.configFile = configFile
|
||||
r.configDir = configDir
|
||||
appCfg := config.New(configFile, configDir, "")
|
||||
appCfg := config.New(configFile, "", "")
|
||||
ngOpts, shardOpts, err := storageEngineOptionsFromConfig(appCfg, debug)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating engine options from config: %v", err)
|
||||
|
@ -138,11 +135,7 @@ func (r *RootModule) GetOrCreateEngine(ctx context.Context, configFile string, c
|
|||
return nil, fmt.Errorf("initializing engine: %v", err)
|
||||
}
|
||||
} else if configFile != r.configFile {
|
||||
return nil, fmt.Errorf("GetOrCreateEngine called with mismatching configFile after engine was "+
|
||||
"initialized: got %q, want %q", configFile, r.configFile)
|
||||
} else if configDir != r.configDir {
|
||||
return nil, fmt.Errorf("GetOrCreateEngine called with mismatching configDir after engine was "+
|
||||
"initialized: got %q, want %q", configDir, r.configDir)
|
||||
return nil, fmt.Errorf("GetOrCreateEngine called with mismatching configFile after engine was initialized: got %q, want %q", configFile, r.configFile)
|
||||
}
|
||||
|
||||
return r.ng, nil
|
||||
|
@ -156,10 +149,10 @@ func (s *Local) Exports() modules.Exports {
|
|||
|
||||
func (s *Local) VU() modules.VU { return s.vu }
|
||||
|
||||
func (s *Local) Connect(configFile, configDir, hexKey string, debug bool) (*Client, error) {
|
||||
ng, err := s.ResolveEngine(s.VU().Context(), configFile, configDir, debug)
|
||||
func (s *Local) Connect(configFile, hexKey string, debug bool) (*Client, error) {
|
||||
ng, err := s.ResolveEngine(s.VU().Context(), configFile, debug)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connecting to engine for config - file %q dir %q: %v", configFile, configDir, err)
|
||||
return nil, fmt.Errorf("connecting to engine for config %q: %v", configFile, err)
|
||||
}
|
||||
|
||||
key, err := ParseOrCreateKey(hexKey)
|
||||
|
|
|
@ -56,7 +56,7 @@ func (s *Local) Exports() modules.Exports {
|
|||
return modules.Exports{Default: s}
|
||||
}
|
||||
|
||||
func (s *Local) Connect(configFile string, configDir string, params map[string]string, bucketMapping map[string]string) (*Client, error) {
|
||||
func (s *Local) Connect(configFile string, params map[string]string, bucketMapping map[string]string) (*Client, error) {
|
||||
// Parse configuration flags.
|
||||
fs := flag.NewFlagSet("s3local", flag.ContinueOnError)
|
||||
|
||||
|
@ -107,9 +107,9 @@ func (s *Local) Connect(configFile string, configDir string, params map[string]s
|
|||
objGetDuration, _ = registry.NewMetric("s3local_obj_get_duration", metrics.Trend, metrics.Time)
|
||||
|
||||
// Create S3 layer backed by local storage engine and tree service.
|
||||
ng, err := s.l.ResolveEngine(s.l.VU().Context(), configFile, configDir, *debugLogger)
|
||||
ng, err := s.l.ResolveEngine(s.l.VU().Context(), configFile, *debugLogger)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connecting to engine for config - file %q dir %q: %v", configFile, configDir, err)
|
||||
return nil, fmt.Errorf("connecting to engine for config %q: %v", configFile, err)
|
||||
}
|
||||
|
||||
treeSvc := tree.NewTree(treeServiceEngineWrapper{
|
||||
|
|
|
@ -49,7 +49,6 @@ if (registry_enabled && delete_age) {
|
|||
);
|
||||
}
|
||||
|
||||
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
|
||||
let obj_to_read_selector = undefined;
|
||||
if (registry_enabled) {
|
||||
obj_to_read_selector = registry.getSelector(
|
||||
|
@ -58,7 +57,6 @@ if (registry_enabled) {
|
|||
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
||||
{
|
||||
status: "created",
|
||||
age: read_age,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
@ -121,17 +119,12 @@ export function setup() {
|
|||
console.log(`Writing VUs: ${write_vu_count}`);
|
||||
console.log(`Deleting VUs: ${delete_vu_count}`);
|
||||
console.log(`Total VUs: ${total_vu_count}`);
|
||||
|
||||
const start_timestamp = Date.now()
|
||||
console.log(`Load started at: ${Date(start_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function teardown(data) {
|
||||
if (obj_registry) {
|
||||
obj_registry.close();
|
||||
}
|
||||
const end_timestamp = Date.now()
|
||||
console.log(`Load finished at: ${Date(end_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function handleSummary(data) {
|
||||
|
|
|
@ -49,7 +49,6 @@ if (registry_enabled && delete_age) {
|
|||
);
|
||||
}
|
||||
|
||||
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
|
||||
let obj_to_read_selector = undefined;
|
||||
if (registry_enabled) {
|
||||
obj_to_read_selector = registry.getSelector(
|
||||
|
@ -58,7 +57,6 @@ if (registry_enabled) {
|
|||
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
||||
{
|
||||
status: "created",
|
||||
age: read_age,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
@ -145,17 +143,12 @@ export function setup() {
|
|||
console.log(`Read rate: ${read_rate}`);
|
||||
console.log(`Writing rate: ${write_rate}`);
|
||||
console.log(`Delete rate: ${delete_rate}`);
|
||||
|
||||
const start_timestamp = Date.now()
|
||||
console.log(`Load started at: ${Date(start_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function teardown(data) {
|
||||
if (obj_registry) {
|
||||
obj_registry.close();
|
||||
}
|
||||
const end_timestamp = Date.now()
|
||||
console.log(`Load finished at: ${Date(end_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function handleSummary(data) {
|
||||
|
|
|
@ -71,17 +71,12 @@ export function setup() {
|
|||
console.log(`Reading VUs: ${read_vu_count}`);
|
||||
console.log(`Writing VUs: ${write_vu_count}`);
|
||||
console.log(`Total VUs: ${total_vu_count}`);
|
||||
|
||||
const start_timestamp = Date.now()
|
||||
console.log(`Load started at: ${Date(start_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function teardown(data) {
|
||||
if (obj_registry) {
|
||||
obj_registry.close();
|
||||
}
|
||||
const end_timestamp = Date.now()
|
||||
console.log(`Load finished at: ${Date(end_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function handleSummary(data) {
|
||||
|
|
|
@ -21,10 +21,9 @@ const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
|
|||
const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
|
||||
|
||||
const config_file = __ENV.CONFIG_FILE;
|
||||
const config_dir = __ENV.CONFIG_DIR;
|
||||
const debug_logger = (__ENV.DEBUG_LOGGER || 'false') == 'true';
|
||||
const local_client = local.connect(config_file, config_dir, '', debug_logger);
|
||||
const log = logging.new().withFields({"config_file": config_file,"config_dir": config_dir});
|
||||
const local_client = local.connect(config_file, '', debug_logger);
|
||||
const log = logging.new().withField("config", config_file);
|
||||
|
||||
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
||||
const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined;
|
||||
|
@ -101,17 +100,12 @@ export function setup() {
|
|||
console.log(`Writing VUs: ${write_vu_count}`);
|
||||
console.log(`Deleting VUs: ${delete_vu_count}`);
|
||||
console.log(`Total VUs: ${total_vu_count}`);
|
||||
|
||||
const start_timestamp = Date.now()
|
||||
console.log(`Load started at: ${Date(start_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function teardown(data) {
|
||||
if (obj_registry) {
|
||||
obj_registry.close();
|
||||
}
|
||||
const end_timestamp = Date.now()
|
||||
console.log(`Load finished at: ${Date(end_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function handleSummary(data) {
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import uuid
|
||||
|
||||
from helpers.cmd import execute_cmd, log
|
||||
from helpers.cmd import execute_cmd
|
||||
|
||||
|
||||
def create_bucket(endpoint, versioning, location, no_verify_ssl):
|
||||
|
@ -13,24 +13,22 @@ def create_bucket(endpoint, versioning, location, no_verify_ssl):
|
|||
cmd_line_ver = f"aws {no_verify_ssl_str} s3api put-bucket-versioning --bucket {bucket_name} " \
|
||||
f"--versioning-configuration Status=Enabled --endpoint {endpoint} "
|
||||
|
||||
output, success = execute_cmd(cmd_line)
|
||||
out, success = execute_cmd(cmd_line)
|
||||
|
||||
if not success and "succeeded and you already own it" not in output:
|
||||
log(f"{cmd_line}\n"
|
||||
f"Bucket {bucket_name} has not been created:\n"
|
||||
f"Error: {output}", endpoint)
|
||||
if not success and "succeeded and you already own it" not in out:
|
||||
print(f" > Bucket {bucket_name} has not been created:\n{out}")
|
||||
return False
|
||||
|
||||
print(f"cmd: {cmd_line}")
|
||||
|
||||
if versioning == "True":
|
||||
output, success = execute_cmd(cmd_line_ver)
|
||||
out, success = execute_cmd(cmd_line_ver)
|
||||
if not success:
|
||||
log(f"{cmd_line_ver}\n"
|
||||
f"Bucket versioning has not been applied for bucket {bucket_name}\n"
|
||||
f"Error: {output}", endpoint)
|
||||
print(f" > Bucket versioning has not been applied for bucket {bucket_name}:\n{out}")
|
||||
else:
|
||||
log(f"Bucket versioning has been applied for bucket {bucket_name}", endpoint)
|
||||
print(f" > Bucket versioning has been applied.")
|
||||
|
||||
log(f"Created bucket: {bucket_name}", endpoint)
|
||||
print(f"Created bucket: {bucket_name} via endpoint {endpoint}")
|
||||
return bucket_name
|
||||
|
||||
|
||||
|
@ -39,12 +37,10 @@ def upload_object(bucket, payload_filepath, endpoint, no_verify_ssl):
|
|||
no_verify_ssl_str = "--no-verify-ssl" if no_verify_ssl else ""
|
||||
cmd_line = f"aws {no_verify_ssl_str} s3api put-object --bucket {bucket} --key {object_name} " \
|
||||
f"--body {payload_filepath} --endpoint {endpoint}"
|
||||
output, success = execute_cmd(cmd_line)
|
||||
out, success = execute_cmd(cmd_line)
|
||||
|
||||
if not success:
|
||||
log(f"{cmd_line}\n"
|
||||
f"Object {object_name} has not been uploaded\n"
|
||||
f"Error: {output}", endpoint)
|
||||
print(f" > Object {object_name} has not been uploaded.")
|
||||
return False
|
||||
|
||||
return bucket, endpoint, object_name
|
||||
|
|
|
@ -1,12 +1,9 @@
|
|||
import os
|
||||
import shlex
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
from subprocess import check_output, CalledProcessError, STDOUT
|
||||
|
||||
def log(message, endpoint):
|
||||
time = datetime.utcnow()
|
||||
print(f"{time} at {endpoint}: {message}")
|
||||
|
||||
def execute_cmd(cmd_line):
|
||||
cmd_args = shlex.split(cmd_line)
|
||||
|
|
|
@ -1,61 +1,47 @@
|
|||
import re
|
||||
from helpers.cmd import execute_cmd, log
|
||||
|
||||
from helpers.cmd import execute_cmd
|
||||
|
||||
|
||||
def create_container(endpoint, policy, wallet_file, wallet_config):
|
||||
if wallet_file:
|
||||
wallet_file = "--wallet " + wallet_file
|
||||
if wallet_config:
|
||||
wallet_config = "--config " + wallet_config
|
||||
cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} container create {wallet_file} {wallet_config} " \
|
||||
cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} container create --wallet {wallet_file} --config {wallet_config} " \
|
||||
f" --policy '{policy}' --basic-acl public-read-write --await"
|
||||
|
||||
output, success = execute_cmd(cmd_line)
|
||||
|
||||
if not success:
|
||||
log(f"{cmd_line}\n"
|
||||
f"Container has not been created\n"
|
||||
f"{output}", endpoint)
|
||||
print(f" > Container has not been created:\n{output}")
|
||||
return False
|
||||
|
||||
try:
|
||||
fst_str = output.split('\n')[0]
|
||||
except Exception:
|
||||
log(f"{cmd_line}\n"
|
||||
f"Incorrect output\n"
|
||||
f"Output: {output or '<empty>'}", endpoint)
|
||||
print(f"Got empty output: {output}")
|
||||
return False
|
||||
splitted = fst_str.split(": ")
|
||||
if len(splitted) != 2:
|
||||
raise ValueError(f"no CID was parsed from command output:\t{fst_str}")
|
||||
raise ValueError(f"no CID was parsed from command output: \t{fst_str}")
|
||||
|
||||
log(f"Created container {splitted[1]}", endpoint)
|
||||
print(f"Created container: {splitted[1]} via endpoint {endpoint}")
|
||||
|
||||
return splitted[1]
|
||||
|
||||
|
||||
def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_config):
|
||||
object_name = ""
|
||||
if wallet_file:
|
||||
wallet_file = "--wallet " + wallet_file
|
||||
if wallet_config:
|
||||
wallet_config = "--config " + wallet_config
|
||||
cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} object put --file {payload_filepath} {wallet_file} {wallet_config} " \
|
||||
cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} object put --file {payload_filepath} --wallet {wallet_file} --config {wallet_config} " \
|
||||
f"--cid {container} --no-progress"
|
||||
output, success = execute_cmd(cmd_line)
|
||||
|
||||
if not success:
|
||||
log(f"{cmd_line}\n"
|
||||
f"Object {object_name} has not been uploaded\n"
|
||||
f"Error: {output}", endpoint)
|
||||
print(f" > Object {object_name} has not been uploaded:\n{output}")
|
||||
return False
|
||||
|
||||
try:
|
||||
# taking second string from command output
|
||||
snd_str = output.split('\n')[1]
|
||||
except Exception:
|
||||
log(f"{cmd_line}\n"
|
||||
f"Incorrect output\n"
|
||||
f"Output: {output or '<empty>'}", endpoint)
|
||||
print(f"Got empty input: {output}")
|
||||
return False
|
||||
splitted = snd_str.split(": ")
|
||||
if len(splitted) != 2:
|
||||
|
@ -64,42 +50,32 @@ def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_con
|
|||
|
||||
|
||||
def get_object(cid, oid, endpoint, out_filepath, wallet_file, wallet_config):
|
||||
if wallet_file:
|
||||
wallet_file = "--wallet " + wallet_file
|
||||
if wallet_config:
|
||||
wallet_config = "--config " + wallet_config
|
||||
cmd_line = f"frostfs-cli object get -r {endpoint} --cid {cid} --oid {oid} {wallet_file} {wallet_config} " \
|
||||
cmd_line = f"frostfs-cli object get -r {endpoint} --cid {cid} --oid {oid} --wallet {wallet_file} --config {wallet_config} " \
|
||||
f"--file {out_filepath}"
|
||||
|
||||
output, success = execute_cmd(cmd_line)
|
||||
|
||||
if not success:
|
||||
log(f"{cmd_line}\n"
|
||||
f"Failed to get object {oid} from container {cid}\n"
|
||||
f"Error: {output}", endpoint)
|
||||
print(f" > Failed to get object {output} from container {cid} \r\n"
|
||||
f" > Error: {output}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def search_object_by_id(cid, oid, endpoint, wallet_file, wallet_config, ttl=2):
|
||||
if wallet_file:
|
||||
wallet_file = "--wallet " + wallet_file
|
||||
if wallet_config:
|
||||
wallet_config = "--config " + wallet_config
|
||||
cmd_line = f"frostfs-cli object search --ttl {ttl} -r {endpoint} --cid {cid} --oid {oid} {wallet_file} {wallet_config} "
|
||||
cmd_line = f"frostfs-cli object search --ttl {ttl} -r {endpoint} --cid {cid} --oid {oid} --wallet {wallet_file} --config {wallet_config} "
|
||||
|
||||
output, success = execute_cmd(cmd_line)
|
||||
|
||||
if not success:
|
||||
log(f"{cmd_line}\n"
|
||||
f"Failed to search object {oid} for container {cid}\n"
|
||||
f"Error: {output}", endpoint)
|
||||
print(f" > Failed to search object {oid} for container {cid} \r\n"
|
||||
f" > Error: {output}")
|
||||
return False
|
||||
|
||||
re_rst = re.search(r'Found (\d+) objects', output)
|
||||
|
||||
if not re_rst:
|
||||
raise Exception("Failed to parse search results")
|
||||
raise Exception("Failed to parce search results")
|
||||
|
||||
return re_rst.group(1)
|
||||
|
|
|
@ -3,6 +3,7 @@
|
|||
import argparse
|
||||
from itertools import cycle
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
@ -32,7 +33,7 @@ parser.add_argument('--endpoint', help='Nodes addresses separated by comma.')
|
|||
parser.add_argument('--update', help='Save existed containers')
|
||||
parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
|
||||
parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Default = 50', default=50)
|
||||
parser.add_argument('--sleep', help='Time to sleep between containers creation and objects upload (in seconds), '
|
||||
parser.add_argument('--sleep', help='Time to sleep between container creation and object PUT (in seconds), '
|
||||
'Default = 8', default=8)
|
||||
|
||||
args: Namespace = parser.parse_args()
|
||||
|
|
|
@ -25,7 +25,7 @@ parser.add_argument('--versioning', help='True/False, False by default.')
|
|||
parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
|
||||
parser.add_argument('--no-verify-ssl', help='Ignore SSL verifications', action='store_true')
|
||||
parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Default = 50', default=50)
|
||||
parser.add_argument('--sleep', help='Time to sleep between buckets creation and objects upload (in seconds), '
|
||||
parser.add_argument('--sleep', help='Time to sleep between container creation and object PUT (in seconds), '
|
||||
'Default = 8', default=8)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
|
|
@ -48,7 +48,6 @@ if (registry_enabled && delete_age) {
|
|||
);
|
||||
}
|
||||
|
||||
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
|
||||
let obj_to_read_selector = undefined;
|
||||
if (registry_enabled) {
|
||||
obj_to_read_selector = registry.getSelector(
|
||||
|
@ -57,7 +56,6 @@ if (registry_enabled) {
|
|||
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
||||
{
|
||||
status: "created",
|
||||
age: read_age,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
@ -118,17 +116,12 @@ export function setup() {
|
|||
console.log(`Writing VUs: ${write_vu_count}`);
|
||||
console.log(`Deleting VUs: ${delete_vu_count}`);
|
||||
console.log(`Total VUs: ${total_vu_count}`);
|
||||
|
||||
const start_timestamp = Date.now()
|
||||
console.log(`Load started at: ${Date(start_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function teardown(data) {
|
||||
if (obj_registry) {
|
||||
obj_registry.close();
|
||||
}
|
||||
const end_timestamp = Date.now()
|
||||
console.log(`Load finished at: ${Date(end_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function handleSummary(data) {
|
||||
|
|
|
@ -48,7 +48,6 @@ if (registry_enabled && delete_age) {
|
|||
);
|
||||
}
|
||||
|
||||
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
|
||||
let obj_to_read_selector = undefined;
|
||||
if (registry_enabled) {
|
||||
obj_to_read_selector = registry.getSelector(
|
||||
|
@ -57,7 +56,6 @@ if (registry_enabled) {
|
|||
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
||||
{
|
||||
status: "created",
|
||||
age: read_age,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
@ -145,17 +143,12 @@ export function setup() {
|
|||
console.log(`Read rate: ${read_rate}`);
|
||||
console.log(`Writing rate: ${write_rate}`);
|
||||
console.log(`Delete rate: ${delete_rate}`);
|
||||
|
||||
const start_timestamp = Date.now()
|
||||
console.log(`Load started at: ${Date(start_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function teardown(data) {
|
||||
if (obj_registry) {
|
||||
obj_registry.close();
|
||||
}
|
||||
const end_timestamp = Date.now()
|
||||
console.log(`Load finished at: ${Date(end_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function handleSummary(data) {
|
||||
|
|
|
@ -19,9 +19,7 @@ const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
|
|||
// Select random S3 endpoint for current VU
|
||||
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
|
||||
const s3_endpoint = s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)];
|
||||
const no_verify_ssl = __ENV.NO_VERIFY_SSL || "true";
|
||||
const connection_args = {no_verify_ssl: no_verify_ssl}
|
||||
const s3_client = s3.connect(s3_endpoint, connection_args);
|
||||
const s3_client = s3.connect(`http://${s3_endpoint}`);
|
||||
const log = logging.new().withField("endpoint", s3_endpoint);
|
||||
|
||||
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
||||
|
|
|
@ -36,11 +36,10 @@ const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
|
|||
const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
|
||||
|
||||
const config_file = __ENV.CONFIG_FILE;
|
||||
const config_dir = __ENV.CONFIG_DIR;
|
||||
const s3_client = s3local.connect(config_file, config_dir, {
|
||||
const s3_client = s3local.connect(config_file, {
|
||||
'debug_logger': __ENV.DEBUG_LOGGER || 'false',
|
||||
}, bucket_mapping());
|
||||
const log = logging.new().withFields({"config_file": config_file,"config_dir": config_dir});
|
||||
const log = logging.new().withField("config", config_file);
|
||||
|
||||
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
||||
const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined;
|
||||
|
@ -99,17 +98,12 @@ export function setup() {
|
|||
console.log(`Reading VUs: ${read_vu_count}`);
|
||||
console.log(`Writing VUs: ${write_vu_count}`);
|
||||
console.log(`Total VUs: ${total_vu_count}`);
|
||||
|
||||
const start_timestamp = Date.now()
|
||||
console.log(`Load started at: ${Date(start_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function teardown(data) {
|
||||
if (obj_registry) {
|
||||
obj_registry.close();
|
||||
}
|
||||
const end_timestamp = Date.now()
|
||||
console.log(`Load finished at: ${Date(end_timestamp).toString()}`)
|
||||
}
|
||||
|
||||
export function handleSummary(data) {
|
||||
|
|
Loading…
Reference in a new issue