forked from TrueCloudLab/xk6-frostfs
Compare commits
12 commits
feat/netwo
...
master
Author | SHA1 | Date | |
---|---|---|---|
c2b8944af6 | |||
a47bf149d8 | |||
bcbd0db25f | |||
17bbbe53e6 | |||
bede693470 | |||
f539da7d89 | |||
6d3ecb6528 | |||
75f670b392 | |||
9b9db46a07 | |||
335c45c578 | |||
e7d4dd404a | |||
0a9aeab47c |
17 changed files with 237 additions and 51 deletions
|
@ -13,7 +13,7 @@ jobs:
|
||||||
- name: Setup Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@v3
|
uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: '1.21'
|
go-version: '1.22'
|
||||||
|
|
||||||
- name: Run commit format checker
|
- name: Run commit format checker
|
||||||
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2
|
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2
|
||||||
|
|
|
@ -11,20 +11,21 @@ jobs:
|
||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: actions/setup-go@v3
|
uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: '1.22'
|
go-version: '1.23'
|
||||||
cache: true
|
cache: true
|
||||||
|
|
||||||
- name: golangci-lint
|
- name: Install linters
|
||||||
uses: https://github.com/golangci/golangci-lint-action@v3
|
run: make lint-install
|
||||||
with:
|
|
||||||
version: latest
|
- name: Run linters
|
||||||
|
run: make lint
|
||||||
|
|
||||||
tests:
|
tests:
|
||||||
name: Tests
|
name: Tests
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go_versions: [ '1.21', '1.22' ]
|
go_versions: [ '1.22', '1.23' ]
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
|
@ -47,7 +48,7 @@ jobs:
|
||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: actions/setup-go@v3
|
uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: '1.21'
|
go-version: '1.22'
|
||||||
cache: true
|
cache: true
|
||||||
|
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
|
|
|
@ -3,8 +3,8 @@
|
||||||
First, thank you for contributing! We love and encourage pull requests from
|
First, thank you for contributing! We love and encourage pull requests from
|
||||||
everyone. Please follow the guidelines:
|
everyone. Please follow the guidelines:
|
||||||
|
|
||||||
- Check the open [issues](https://github.com/TrueCloudLab/xk6-frostfs/issues) and
|
- Check the open [issues](https://git.frostfs.info/TrueCloudLab/xk6-frostfs/issues) and
|
||||||
[pull requests](https://github.com/TrueCloudLab/xk6-frostfs/pulls) for existing
|
[pull requests](https://git.frostfs.info/TrueCloudLab/xk6-frostfs/pulls) for existing
|
||||||
discussions.
|
discussions.
|
||||||
|
|
||||||
- Open an issue first, to discuss a new feature or enhancement.
|
- Open an issue first, to discuss a new feature or enhancement.
|
||||||
|
@ -28,18 +28,18 @@ send a pull request. We encourage pull requests to discuss code changes. Here
|
||||||
are the steps in details:
|
are the steps in details:
|
||||||
|
|
||||||
### Set up your GitHub Repository
|
### Set up your GitHub Repository
|
||||||
Fork [xk6-frostfs upstream](https://github.com/TrueCloudLab/xk6-frostfs/fork) source
|
Fork [xk6-frostfs upstream](https://git.frostfs.info/TrueCloudLab/xk6-frostfs/fork) source
|
||||||
repository to your own personal repository. Copy the URL of your fork (you will
|
repository to your own personal repository. Copy the URL of your fork (you will
|
||||||
need it for the `git clone` command below).
|
need it for the `git clone` command below).
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
$ git clone https://github.com/TrueCloudLab/xk6-frostfs
|
$ git clone https://git.frostfs.info/TrueCloudLab/xk6-frostfs
|
||||||
```
|
```
|
||||||
|
|
||||||
### Set up git remote as ``upstream``
|
### Set up git remote as ``upstream``
|
||||||
```sh
|
```sh
|
||||||
$ cd xk6-frostfs
|
$ cd xk6-frostfs
|
||||||
$ git remote add upstream https://github.com/TrueCloudLab/xk6-frostfs
|
$ git remote add upstream https://git.frostfs.info/TrueCloudLab/xk6-frostfs
|
||||||
$ git fetch upstream
|
$ git fetch upstream
|
||||||
$ git merge upstream/master
|
$ git merge upstream/master
|
||||||
...
|
...
|
||||||
|
|
26
Makefile
26
Makefile
|
@ -3,10 +3,15 @@
|
||||||
# Common variables
|
# Common variables
|
||||||
REPO ?= $(shell go list -m)
|
REPO ?= $(shell go list -m)
|
||||||
VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")
|
VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")
|
||||||
GO_VERSION ?= 1.19
|
GO_VERSION ?= 1.22
|
||||||
LINT_VERSION ?= 1.49.0
|
LINT_VERSION ?= 1.60.3
|
||||||
|
TRUECLOUDLAB_LINT_VERSION ?= 0.0.7
|
||||||
BINDIR = bin
|
BINDIR = bin
|
||||||
|
|
||||||
|
OUTPUT_LINT_DIR ?= $(abspath $(BINDIR))/linters
|
||||||
|
LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
|
||||||
|
TMP_DIR := .cache
|
||||||
|
|
||||||
# Binaries to build
|
# Binaries to build
|
||||||
CMDS = $(addprefix frostfs-, $(notdir $(wildcard cmd/*)))
|
CMDS = $(addprefix frostfs-, $(notdir $(wildcard cmd/*)))
|
||||||
BINS = $(addprefix $(BINDIR)/, $(CMDS))
|
BINS = $(addprefix $(BINDIR)/, $(CMDS))
|
||||||
|
@ -64,7 +69,22 @@ format:
|
||||||
|
|
||||||
# Run linters
|
# Run linters
|
||||||
lint:
|
lint:
|
||||||
@golangci-lint --timeout=5m run
|
@if [ ! -d "$(LINT_DIR)" ]; then \
|
||||||
|
make lint-install; \
|
||||||
|
fi
|
||||||
|
$(LINT_DIR)/golangci-lint run --timeout=5m
|
||||||
|
|
||||||
|
# Install linters
|
||||||
|
lint-install:
|
||||||
|
@rm -rf $(OUTPUT_LINT_DIR)
|
||||||
|
@mkdir -p $(OUTPUT_LINT_DIR)
|
||||||
|
@mkdir -p $(TMP_DIR)
|
||||||
|
@rm -rf $(TMP_DIR)/linters
|
||||||
|
@git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters
|
||||||
|
@@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR)
|
||||||
|
@rm -rf $(TMP_DIR)/linters
|
||||||
|
@rmdir $(TMP_DIR) 2>/dev/null || true
|
||||||
|
@CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install -trimpath github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION)
|
||||||
|
|
||||||
# Run linters in Docker
|
# Run linters in Docker
|
||||||
docker/lint:
|
docker/lint:
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
|
<img src="./.forgejo/logo.svg" width="500px" alt="FrostFS logo">
|
||||||
</p>
|
</p>
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<a href="https://go.k6.io/k6">k6</a> extension to test and benchmark FrostFS related protocols.
|
<a href="https://go.k6.io/k6">k6</a> extension to test and benchmark FrostFS related protocols.
|
||||||
|
|
55
cmd/xk6-registry/importer/import.go
Normal file
55
cmd/xk6-registry/importer/import.go
Normal file
|
@ -0,0 +1,55 @@
|
||||||
|
package importer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/registry"
|
||||||
|
)
|
||||||
|
|
||||||
|
type PreGenObj struct {
|
||||||
|
Bucket string `json:"bucket"`
|
||||||
|
Object string `json:"object"`
|
||||||
|
Container string `json:"container"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type PreGenerateInfo struct {
|
||||||
|
Buckets []string `json:"buckets"`
|
||||||
|
Containers []string `json:"containers"`
|
||||||
|
Objects []PreGenObj `json:"objects"`
|
||||||
|
ObjSize string `json:"obj_size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ImportJSONPreGen writes objects from pregenerated JSON file
|
||||||
|
// to the registry.
|
||||||
|
// Note that ImportJSONPreGen does not check if object already
|
||||||
|
// exists in the registry so in case of re-entry the registry
|
||||||
|
// will have two entities representing the same object.
|
||||||
|
func ImportJSONPreGen(o *registry.ObjRegistry, filename string) error {
|
||||||
|
f, err := os.ReadFile(filename)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var pregenInfo PreGenerateInfo
|
||||||
|
err = json.Unmarshal(f, &pregenInfo)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddObject uses DB.Batch to combine concurrent Batch calls
|
||||||
|
// into a single Bolt transaction. DB.Batch is limited by
|
||||||
|
// DB.MaxBatchDelay which may affect perfomance.
|
||||||
|
for _, obj := range pregenInfo.Objects {
|
||||||
|
if obj.Bucket != "" {
|
||||||
|
err = o.AddObject("", "", obj.Bucket, obj.Object, "")
|
||||||
|
} else {
|
||||||
|
err = o.AddObject(obj.Container, obj.Object, "", "", "")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
27
cmd/xk6-registry/importer/root.go
Normal file
27
cmd/xk6-registry/importer/root.go
Normal file
|
@ -0,0 +1,27 @@
|
||||||
|
package importer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/registry"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Cmd represents the import command.
|
||||||
|
var Cmd = &cobra.Command{
|
||||||
|
Use: "import",
|
||||||
|
Short: "Import objects into registry",
|
||||||
|
Long: "Import objects into registry from pregenerated files",
|
||||||
|
Example: `xk6-registry import registry.bolt preset.json
|
||||||
|
xk6-registry import --status created registry.bolt preset.json another_preset.json`,
|
||||||
|
RunE: runCmd,
|
||||||
|
Args: cobra.MinimumNArgs(2),
|
||||||
|
}
|
||||||
|
|
||||||
|
func runCmd(cmd *cobra.Command, args []string) error {
|
||||||
|
objRegistry := registry.NewObjRegistry(cmd.Context(), args[0])
|
||||||
|
for i := 1; i < len(args); i++ {
|
||||||
|
if err := ImportJSONPreGen(objRegistry, args[i]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
18
cmd/xk6-registry/main.go
Normal file
18
cmd/xk6-registry/main.go
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
|
||||||
|
|
||||||
|
if cmd, err := rootCmd.ExecuteContextC(ctx); err != nil {
|
||||||
|
cmd.PrintErrln("Error:", err.Error())
|
||||||
|
cmd.PrintErrf("Run '%v --help' for usage.\n", cmd.CommandPath())
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
33
cmd/xk6-registry/root.go
Normal file
33
cmd/xk6-registry/root.go
Normal file
|
@ -0,0 +1,33 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"runtime"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/xk6-frostfs/cmd/xk6-registry/importer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/version"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var rootCmd = &cobra.Command{
|
||||||
|
Use: "xk6-registry",
|
||||||
|
Version: version.Version,
|
||||||
|
Short: "Command Line Tool to work with Registry",
|
||||||
|
Long: `Registry provides tools to work with object registry for xk6.
|
||||||
|
It contains command for importing objects in registry from preset`,
|
||||||
|
SilenceErrors: true,
|
||||||
|
SilenceUsage: true,
|
||||||
|
Run: rootCmdRun,
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
cobra.AddTemplateFunc("runtimeVersion", runtime.Version)
|
||||||
|
rootCmd.SetVersionTemplate(`FrostFS xk6-registry
|
||||||
|
{{printf "Version: %s" .Version }}
|
||||||
|
GoVersion: {{ runtimeVersion }}
|
||||||
|
`)
|
||||||
|
rootCmd.AddCommand(importer.Cmd)
|
||||||
|
}
|
||||||
|
|
||||||
|
func rootCmdRun(cmd *cobra.Command, _ []string) {
|
||||||
|
_ = cmd.Usage()
|
||||||
|
}
|
2
go.mod
2
go.mod
|
@ -1,6 +1,6 @@
|
||||||
module git.frostfs.info/TrueCloudLab/xk6-frostfs
|
module git.frostfs.info/TrueCloudLab/xk6-frostfs
|
||||||
|
|
||||||
go 1.21
|
go 1.22
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-node v0.38.3-0.20240502170333-ec2873caa7c6
|
git.frostfs.info/TrueCloudLab/frostfs-node v0.38.3-0.20240502170333-ec2873caa7c6
|
||||||
|
|
|
@ -3,12 +3,15 @@ package registry
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neo-go/pkg/io"
|
"github.com/nspcc-dev/neo-go/pkg/io"
|
||||||
"go.etcd.io/bbolt"
|
"go.etcd.io/bbolt"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const nextObjectTimeout = 10 * time.Second
|
||||||
|
|
||||||
type ObjFilter struct {
|
type ObjFilter struct {
|
||||||
Status string
|
Status string
|
||||||
Age int
|
Age int
|
||||||
|
@ -21,6 +24,8 @@ type ObjSelector struct {
|
||||||
filter *ObjFilter
|
filter *ObjFilter
|
||||||
cacheSize int
|
cacheSize int
|
||||||
kind SelectorKind
|
kind SelectorKind
|
||||||
|
// Sync synchronizes VU used for deletion.
|
||||||
|
Sync sync.WaitGroup
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectSelectCache is the default maximum size of a batch to select from DB.
|
// objectSelectCache is the default maximum size of a batch to select from DB.
|
||||||
|
@ -57,7 +62,16 @@ func NewObjSelector(registry *ObjRegistry, selectionSize int, kind SelectorKind,
|
||||||
// - underlying registry context is done, nil objects will be returned on the
|
// - underlying registry context is done, nil objects will be returned on the
|
||||||
// currently blocked and every further NextObject calls.
|
// currently blocked and every further NextObject calls.
|
||||||
func (o *ObjSelector) NextObject() *ObjectInfo {
|
func (o *ObjSelector) NextObject() *ObjectInfo {
|
||||||
return <-o.objChan
|
if o.kind == SelectorOneshot {
|
||||||
|
return <-o.objChan
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-time.After(nextObjectTimeout):
|
||||||
|
return nil
|
||||||
|
case obj := <-o.objChan:
|
||||||
|
return obj
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count returns total number of objects that match filter of the selector.
|
// Count returns total number of objects that match filter of the selector.
|
||||||
|
|
|
@ -4,15 +4,16 @@ from helpers.cmd import execute_cmd, log
|
||||||
|
|
||||||
|
|
||||||
def create_bucket(endpoint, versioning, location, acl, no_verify_ssl):
|
def create_bucket(endpoint, versioning, location, acl, no_verify_ssl):
|
||||||
|
configuration = ""
|
||||||
if location:
|
if location:
|
||||||
location = f"--create-bucket-configuration 'LocationConstraint={location}'"
|
configuration = f"--create-bucket-configuration 'LocationConstraint={location}'"
|
||||||
if acl:
|
if acl:
|
||||||
acl = f"--acl {acl}"
|
acl = f"--acl {acl}"
|
||||||
|
|
||||||
bucket_name = str(uuid.uuid4())
|
bucket_name = str(uuid.uuid4())
|
||||||
no_verify_ssl_str = "--no-verify-ssl" if no_verify_ssl else ""
|
no_verify_ssl_str = "--no-verify-ssl" if no_verify_ssl else ""
|
||||||
cmd_line = f"aws {no_verify_ssl_str} s3api create-bucket --bucket {bucket_name} " \
|
cmd_line = f"aws {no_verify_ssl_str} s3api create-bucket --bucket {bucket_name} " \
|
||||||
f"--endpoint {endpoint} {location} {acl} "
|
f"--endpoint {endpoint} {configuration} {acl} "
|
||||||
cmd_line_ver = f"aws {no_verify_ssl_str} s3api put-bucket-versioning --bucket {bucket_name} " \
|
cmd_line_ver = f"aws {no_verify_ssl_str} s3api put-bucket-versioning --bucket {bucket_name} " \
|
||||||
f"--versioning-configuration Status=Enabled --endpoint {endpoint} {acl} "
|
f"--versioning-configuration Status=Enabled --endpoint {endpoint} {acl} "
|
||||||
|
|
||||||
|
@ -33,7 +34,7 @@ def create_bucket(endpoint, versioning, location, acl, no_verify_ssl):
|
||||||
else:
|
else:
|
||||||
log(f"Bucket versioning has been applied for bucket {bucket_name}", endpoint)
|
log(f"Bucket versioning has been applied for bucket {bucket_name}", endpoint)
|
||||||
|
|
||||||
log(f"Created bucket: {bucket_name}", endpoint)
|
log(f"Created bucket: {bucket_name} ({location})", endpoint)
|
||||||
return bucket_name
|
return bucket_name
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
import re
|
import re
|
||||||
from helpers.cmd import execute_cmd, log
|
from helpers.cmd import execute_cmd, log
|
||||||
|
|
||||||
def create_container(endpoint, policy, wallet_path, config, acl, local=False, depth=0):
|
def create_container(endpoint, policy, container_creation_retry, wallet_path, config, acl, local=False, retry=0):
|
||||||
if depth > 20:
|
if retry > int(container_creation_retry):
|
||||||
raise ValueError(f"unable to create container: too many unsuccessful attempts")
|
raise ValueError(f"unable to create container: too many unsuccessful attempts")
|
||||||
|
|
||||||
if wallet_path:
|
if wallet_path:
|
||||||
|
@ -34,7 +34,7 @@ def create_container(endpoint, policy, wallet_path, config, acl, local=False, de
|
||||||
raise ValueError(f"no CID was parsed from command output:\t{fst_str}")
|
raise ValueError(f"no CID was parsed from command output:\t{fst_str}")
|
||||||
cid = splitted[1]
|
cid = splitted[1]
|
||||||
|
|
||||||
log(f"Created container {cid}", endpoint)
|
log(f"Created container: {cid} ({policy})", endpoint)
|
||||||
|
|
||||||
if not local:
|
if not local:
|
||||||
return cid
|
return cid
|
||||||
|
@ -88,7 +88,7 @@ def create_container(endpoint, policy, wallet_path, config, acl, local=False, de
|
||||||
return cid
|
return cid
|
||||||
|
|
||||||
log(f"Created container {cid} is not stored on {endpoint}, creating another one...", endpoint)
|
log(f"Created container {cid} is not stored on {endpoint}, creating another one...", endpoint)
|
||||||
return create_container(endpoint, policy, wallet_path, config, acl, local, depth + 1)
|
return create_container(endpoint, policy, container_creation_retry, wallet_path, config, acl, local, retry + 1)
|
||||||
|
|
||||||
|
|
||||||
def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_config):
|
def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_config):
|
||||||
|
|
|
@ -15,18 +15,20 @@ from helpers.frostfs_cli import create_container, upload_object
|
||||||
ERROR_WRONG_CONTAINERS_COUNT = 1
|
ERROR_WRONG_CONTAINERS_COUNT = 1
|
||||||
ERROR_WRONG_OBJECTS_COUNT = 2
|
ERROR_WRONG_OBJECTS_COUNT = 2
|
||||||
MAX_WORKERS = 50
|
MAX_WORKERS = 50
|
||||||
|
DEFAULT_POLICY = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument('--size', help='Upload objects size in kb')
|
parser.add_argument('--size', help='Upload objects size in kb')
|
||||||
parser.add_argument('--containers', help='Number of containers to create')
|
parser.add_argument('--containers', help='Number of containers to create')
|
||||||
|
parser.add_argument('--retry', default=20, help='Maximum number of retries to create a container')
|
||||||
parser.add_argument('--out', help='JSON file with output')
|
parser.add_argument('--out', help='JSON file with output')
|
||||||
parser.add_argument('--preload_obj', help='Number of pre-loaded objects')
|
parser.add_argument('--preload_obj', help='Number of pre-loaded objects')
|
||||||
parser.add_argument('--wallet', help='Wallet file path')
|
parser.add_argument('--wallet', help='Wallet file path')
|
||||||
parser.add_argument('--config', help='Wallet config file path')
|
parser.add_argument('--config', help='Wallet config file path')
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--policy",
|
"--policy",
|
||||||
help="Container placement policy",
|
help=f"Container placement policy. Default is {DEFAULT_POLICY}",
|
||||||
default="REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
|
action="append"
|
||||||
)
|
)
|
||||||
parser.add_argument('--endpoint', help='Nodes addresses separated by comma.')
|
parser.add_argument('--endpoint', help='Nodes addresses separated by comma.')
|
||||||
parser.add_argument('--update', help='Save existed containers')
|
parser.add_argument('--update', help='Save existed containers')
|
||||||
|
@ -46,7 +48,10 @@ def main():
|
||||||
objects_list = []
|
objects_list = []
|
||||||
|
|
||||||
endpoints = args.endpoint.split(',')
|
endpoints = args.endpoint.split(',')
|
||||||
|
if not args.policy:
|
||||||
|
args.policy = [DEFAULT_POLICY]
|
||||||
|
|
||||||
|
container_creation_retry = args.retry
|
||||||
wallet = args.wallet
|
wallet = args.wallet
|
||||||
wallet_config = args.config
|
wallet_config = args.config
|
||||||
workers = int(args.workers)
|
workers = int(args.workers)
|
||||||
|
@ -63,9 +68,9 @@ def main():
|
||||||
containers_count = int(args.containers)
|
containers_count = int(args.containers)
|
||||||
print(f"Create containers: {containers_count}")
|
print(f"Create containers: {containers_count}")
|
||||||
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
|
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
|
||||||
containers_runs = [executor.submit(create_container, endpoint, args.policy, wallet, wallet_config, args.acl, args.local)
|
containers_runs = [executor.submit(create_container, endpoint, policy, container_creation_retry, wallet, wallet_config, args.acl, args.local)
|
||||||
for _, endpoint in
|
for _, endpoint, policy in
|
||||||
zip(range(containers_count), cycle(endpoints))]
|
zip(range(containers_count), cycle(endpoints), cycle(args.policy))]
|
||||||
|
|
||||||
for run in containers_runs:
|
for run in containers_runs:
|
||||||
container_id = run.result()
|
container_id = run.result()
|
||||||
|
|
|
@ -11,6 +11,11 @@ from concurrent.futures import ProcessPoolExecutor
|
||||||
from helpers.cmd import random_payload
|
from helpers.cmd import random_payload
|
||||||
from helpers.aws_cli import create_bucket, upload_object
|
from helpers.aws_cli import create_bucket, upload_object
|
||||||
|
|
||||||
|
ERROR_WRONG_CONTAINERS_COUNT = 1
|
||||||
|
ERROR_WRONG_OBJECTS_COUNT = 2
|
||||||
|
MAX_WORKERS = 50
|
||||||
|
DEFAULT_LOCATION = ""
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
|
|
||||||
parser.add_argument('--size', help='Upload objects size in kb.')
|
parser.add_argument('--size', help='Upload objects size in kb.')
|
||||||
|
@ -20,7 +25,7 @@ parser.add_argument('--preload_obj', help='Number of pre-loaded objects.')
|
||||||
parser.add_argument('--endpoint', help='S3 Gateways addresses separated by comma.')
|
parser.add_argument('--endpoint', help='S3 Gateways addresses separated by comma.')
|
||||||
parser.add_argument('--update', help='True/False, False by default. Save existed buckets from target file (--out). '
|
parser.add_argument('--update', help='True/False, False by default. Save existed buckets from target file (--out). '
|
||||||
'New buckets will not be created.')
|
'New buckets will not be created.')
|
||||||
parser.add_argument('--location', help='AWS location. Will be empty, if has not be declared.', default="")
|
parser.add_argument('--location', help=f'AWS location constraint. Default is "{DEFAULT_LOCATION}"', action="append")
|
||||||
parser.add_argument('--versioning', help='True/False, False by default.')
|
parser.add_argument('--versioning', help='True/False, False by default.')
|
||||||
parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
|
parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
|
||||||
parser.add_argument('--no-verify-ssl', help='Ignore SSL verifications', action='store_true')
|
parser.add_argument('--no-verify-ssl', help='Ignore SSL verifications', action='store_true')
|
||||||
|
@ -32,10 +37,6 @@ parser.add_argument('--acl', help='Bucket ACL. Default is private. Expected valu
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
print(args)
|
print(args)
|
||||||
|
|
||||||
ERROR_WRONG_CONTAINERS_COUNT = 1
|
|
||||||
ERROR_WRONG_OBJECTS_COUNT = 2
|
|
||||||
MAX_WORKERS = 50
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
buckets = []
|
buckets = []
|
||||||
objects_list = []
|
objects_list = []
|
||||||
|
@ -43,6 +44,8 @@ def main():
|
||||||
no_verify_ssl = args.no_verify_ssl
|
no_verify_ssl = args.no_verify_ssl
|
||||||
|
|
||||||
endpoints = args.endpoint.split(',')
|
endpoints = args.endpoint.split(',')
|
||||||
|
if not args.location:
|
||||||
|
args.location = [DEFAULT_LOCATION]
|
||||||
|
|
||||||
workers = int(args.workers)
|
workers = int(args.workers)
|
||||||
objects_per_bucket = int(args.preload_obj)
|
objects_per_bucket = int(args.preload_obj)
|
||||||
|
@ -59,9 +62,9 @@ def main():
|
||||||
print(f"Create buckets: {buckets_count}")
|
print(f"Create buckets: {buckets_count}")
|
||||||
|
|
||||||
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
|
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
|
||||||
buckets_runs = [executor.submit(create_bucket, endpoint, args.versioning, args.location, args.acl, no_verify_ssl)
|
buckets_runs = [executor.submit(create_bucket, endpoint, args.versioning, location, args.acl, no_verify_ssl)
|
||||||
for _, endpoint in
|
for _, endpoint, location in
|
||||||
zip(range(buckets_count), cycle(endpoints))]
|
zip(range(buckets_count), cycle(endpoints), cycle(args.location))]
|
||||||
|
|
||||||
for run in buckets_runs:
|
for run in buckets_runs:
|
||||||
bucket_name = run.result()
|
bucket_name = run.result()
|
||||||
|
|
|
@ -132,6 +132,10 @@ export function setup() {
|
||||||
const start_timestamp = Date.now()
|
const start_timestamp = Date.now()
|
||||||
console.log(
|
console.log(
|
||||||
`Load started at: ${Date(start_timestamp).toString()}`)
|
`Load started at: ${Date(start_timestamp).toString()}`)
|
||||||
|
|
||||||
|
if (delete_vu_count > 0){
|
||||||
|
obj_to_delete_selector.sync.add(delete_vu_count)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function teardown(data) {
|
export function teardown(data) {
|
||||||
|
@ -204,6 +208,8 @@ export function obj_delete() {
|
||||||
const obj = obj_to_delete_selector.nextObject();
|
const obj = obj_to_delete_selector.nextObject();
|
||||||
if (!obj) {
|
if (!obj) {
|
||||||
if (obj_to_delete_exit_on_null) {
|
if (obj_to_delete_exit_on_null) {
|
||||||
|
obj_to_delete_selector.sync.done()
|
||||||
|
obj_to_delete_selector.sync.wait()
|
||||||
exec.test.abort("No more objects to select");
|
exec.test.abort("No more objects to select");
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
|
|
|
@ -71,11 +71,23 @@ if (write_vu_count > 0) {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const read_vu_count = parseInt(__ENV.READERS || '0');
|
||||||
|
if (read_vu_count > 0) {
|
||||||
|
scenarios.read = {
|
||||||
|
executor : 'constant-vus',
|
||||||
|
vus : read_vu_count,
|
||||||
|
duration : `${duration}s`,
|
||||||
|
exec : 'obj_read',
|
||||||
|
gracefulStop : '5s',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined;
|
const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined;
|
||||||
let obj_to_delete_selector = undefined;
|
let obj_to_delete_selector = undefined;
|
||||||
let obj_to_delete_exit_on_null = undefined;
|
let obj_to_delete_exit_on_null = undefined;
|
||||||
if (registry_enabled && delete_age) {
|
|
||||||
obj_to_delete_exit_on_null = write_vu_count == 0;
|
if (registry_enabled ) {
|
||||||
|
obj_to_delete_exit_on_null = (write_vu_count == 0) && (read_vu_count == 0)
|
||||||
|
|
||||||
let constructor = obj_to_delete_exit_on_null ? registry.getOneshotSelector
|
let constructor = obj_to_delete_exit_on_null ? registry.getOneshotSelector
|
||||||
: registry.getSelector;
|
: registry.getSelector;
|
||||||
|
@ -88,16 +100,7 @@ if (registry_enabled && delete_age) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const read_vu_count = parseInt(__ENV.READERS || '0');
|
|
||||||
if (read_vu_count > 0) {
|
|
||||||
scenarios.read = {
|
|
||||||
executor : 'constant-vus',
|
|
||||||
vus : read_vu_count,
|
|
||||||
duration : `${duration}s`,
|
|
||||||
exec : 'obj_read',
|
|
||||||
gracefulStop : '5s',
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const delete_vu_count = parseInt(__ENV.DELETERS || '0');
|
const delete_vu_count = parseInt(__ENV.DELETERS || '0');
|
||||||
if (delete_vu_count > 0) {
|
if (delete_vu_count > 0) {
|
||||||
|
|
Loading…
Reference in a new issue