Compare commits
No commits in common. "master" and "master" have entirely different histories.
10 changed files with 10 additions and 154 deletions
|
@ -3,8 +3,8 @@
|
||||||
First, thank you for contributing! We love and encourage pull requests from
|
First, thank you for contributing! We love and encourage pull requests from
|
||||||
everyone. Please follow the guidelines:
|
everyone. Please follow the guidelines:
|
||||||
|
|
||||||
- Check the open [issues](https://git.frostfs.info/TrueCloudLab/xk6-frostfs/issues) and
|
- Check the open [issues](https://github.com/TrueCloudLab/xk6-frostfs/issues) and
|
||||||
[pull requests](https://git.frostfs.info/TrueCloudLab/xk6-frostfs/pulls) for existing
|
[pull requests](https://github.com/TrueCloudLab/xk6-frostfs/pulls) for existing
|
||||||
discussions.
|
discussions.
|
||||||
|
|
||||||
- Open an issue first, to discuss a new feature or enhancement.
|
- Open an issue first, to discuss a new feature or enhancement.
|
||||||
|
@ -28,18 +28,18 @@ send a pull request. We encourage pull requests to discuss code changes. Here
|
||||||
are the steps in details:
|
are the steps in details:
|
||||||
|
|
||||||
### Set up your GitHub Repository
|
### Set up your GitHub Repository
|
||||||
Fork [xk6-frostfs upstream](https://git.frostfs.info/TrueCloudLab/xk6-frostfs/fork) source
|
Fork [xk6-frostfs upstream](https://github.com/TrueCloudLab/xk6-frostfs/fork) source
|
||||||
repository to your own personal repository. Copy the URL of your fork (you will
|
repository to your own personal repository. Copy the URL of your fork (you will
|
||||||
need it for the `git clone` command below).
|
need it for the `git clone` command below).
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
$ git clone https://git.frostfs.info/TrueCloudLab/xk6-frostfs
|
$ git clone https://github.com/TrueCloudLab/xk6-frostfs
|
||||||
```
|
```
|
||||||
|
|
||||||
### Set up git remote as ``upstream``
|
### Set up git remote as ``upstream``
|
||||||
```sh
|
```sh
|
||||||
$ cd xk6-frostfs
|
$ cd xk6-frostfs
|
||||||
$ git remote add upstream https://git.frostfs.info/TrueCloudLab/xk6-frostfs
|
$ git remote add upstream https://github.com/TrueCloudLab/xk6-frostfs
|
||||||
$ git fetch upstream
|
$ git fetch upstream
|
||||||
$ git merge upstream/master
|
$ git merge upstream/master
|
||||||
...
|
...
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<img src="./.forgejo/logo.svg" width="500px" alt="FrostFS logo">
|
<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
|
||||||
</p>
|
</p>
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<a href="https://go.k6.io/k6">k6</a> extension to test and benchmark FrostFS related protocols.
|
<a href="https://go.k6.io/k6">k6</a> extension to test and benchmark FrostFS related protocols.
|
||||||
|
|
|
@ -1,55 +0,0 @@
|
||||||
package importer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/registry"
|
|
||||||
)
|
|
||||||
|
|
||||||
type PreGenObj struct {
|
|
||||||
Bucket string `json:"bucket"`
|
|
||||||
Object string `json:"object"`
|
|
||||||
Container string `json:"container"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type PreGenerateInfo struct {
|
|
||||||
Buckets []string `json:"buckets"`
|
|
||||||
Containers []string `json:"containers"`
|
|
||||||
Objects []PreGenObj `json:"objects"`
|
|
||||||
ObjSize string `json:"obj_size"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ImportJSONPreGen writes objects from pregenerated JSON file
|
|
||||||
// to the registry.
|
|
||||||
// Note that ImportJSONPreGen does not check if object already
|
|
||||||
// exists in the registry so in case of re-entry the registry
|
|
||||||
// will have two entities representing the same object.
|
|
||||||
func ImportJSONPreGen(o *registry.ObjRegistry, filename string) error {
|
|
||||||
f, err := os.ReadFile(filename)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var pregenInfo PreGenerateInfo
|
|
||||||
err = json.Unmarshal(f, &pregenInfo)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddObject uses DB.Batch to combine concurrent Batch calls
|
|
||||||
// into a single Bolt transaction. DB.Batch is limited by
|
|
||||||
// DB.MaxBatchDelay which may affect perfomance.
|
|
||||||
for _, obj := range pregenInfo.Objects {
|
|
||||||
if obj.Bucket != "" {
|
|
||||||
err = o.AddObject("", "", obj.Bucket, obj.Object, "")
|
|
||||||
} else {
|
|
||||||
err = o.AddObject(obj.Container, obj.Object, "", "", "")
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
|
@ -1,27 +0,0 @@
|
||||||
package importer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/registry"
|
|
||||||
"github.com/spf13/cobra"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Cmd represents the import command.
|
|
||||||
var Cmd = &cobra.Command{
|
|
||||||
Use: "import",
|
|
||||||
Short: "Import objects into registry",
|
|
||||||
Long: "Import objects into registry from pregenerated files",
|
|
||||||
Example: `xk6-registry import registry.bolt preset.json
|
|
||||||
xk6-registry import --status created registry.bolt preset.json another_preset.json`,
|
|
||||||
RunE: runCmd,
|
|
||||||
Args: cobra.MinimumNArgs(2),
|
|
||||||
}
|
|
||||||
|
|
||||||
func runCmd(cmd *cobra.Command, args []string) error {
|
|
||||||
objRegistry := registry.NewObjRegistry(cmd.Context(), args[0])
|
|
||||||
for i := 1; i < len(args); i++ {
|
|
||||||
if err := ImportJSONPreGen(objRegistry, args[i]); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
|
@ -1,18 +0,0 @@
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"os"
|
|
||||||
"os/signal"
|
|
||||||
"syscall"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
|
|
||||||
|
|
||||||
if cmd, err := rootCmd.ExecuteContextC(ctx); err != nil {
|
|
||||||
cmd.PrintErrln("Error:", err.Error())
|
|
||||||
cmd.PrintErrf("Run '%v --help' for usage.\n", cmd.CommandPath())
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,33 +0,0 @@
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"runtime"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/xk6-frostfs/cmd/xk6-registry/importer"
|
|
||||||
"git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/version"
|
|
||||||
"github.com/spf13/cobra"
|
|
||||||
)
|
|
||||||
|
|
||||||
var rootCmd = &cobra.Command{
|
|
||||||
Use: "xk6-registry",
|
|
||||||
Version: version.Version,
|
|
||||||
Short: "Command Line Tool to work with Registry",
|
|
||||||
Long: `Registry provides tools to work with object registry for xk6.
|
|
||||||
It contains command for importing objects in registry from preset`,
|
|
||||||
SilenceErrors: true,
|
|
||||||
SilenceUsage: true,
|
|
||||||
Run: rootCmdRun,
|
|
||||||
}
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
cobra.AddTemplateFunc("runtimeVersion", runtime.Version)
|
|
||||||
rootCmd.SetVersionTemplate(`FrostFS xk6-registry
|
|
||||||
{{printf "Version: %s" .Version }}
|
|
||||||
GoVersion: {{ runtimeVersion }}
|
|
||||||
`)
|
|
||||||
rootCmd.AddCommand(importer.Cmd)
|
|
||||||
}
|
|
||||||
|
|
||||||
func rootCmdRun(cmd *cobra.Command, _ []string) {
|
|
||||||
_ = cmd.Usage()
|
|
||||||
}
|
|
|
@ -3,7 +3,6 @@ package registry
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neo-go/pkg/io"
|
"github.com/nspcc-dev/neo-go/pkg/io"
|
||||||
|
@ -24,8 +23,6 @@ type ObjSelector struct {
|
||||||
filter *ObjFilter
|
filter *ObjFilter
|
||||||
cacheSize int
|
cacheSize int
|
||||||
kind SelectorKind
|
kind SelectorKind
|
||||||
// Sync synchronizes VU used for deletion.
|
|
||||||
Sync sync.WaitGroup
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectSelectCache is the default maximum size of a batch to select from DB.
|
// objectSelectCache is the default maximum size of a batch to select from DB.
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
import re
|
import re
|
||||||
from helpers.cmd import execute_cmd, log
|
from helpers.cmd import execute_cmd, log
|
||||||
|
|
||||||
def create_container(endpoint, policy, container_creation_retry, wallet_path, config, acl, local=False, retry=0):
|
def create_container(endpoint, policy, wallet_path, config, acl, local=False, depth=0):
|
||||||
if retry > int(container_creation_retry):
|
if depth > 20:
|
||||||
raise ValueError(f"unable to create container: too many unsuccessful attempts")
|
raise ValueError(f"unable to create container: too many unsuccessful attempts")
|
||||||
|
|
||||||
if wallet_path:
|
if wallet_path:
|
||||||
|
@ -88,7 +88,7 @@ def create_container(endpoint, policy, container_creation_retry, wallet_path, co
|
||||||
return cid
|
return cid
|
||||||
|
|
||||||
log(f"Created container {cid} is not stored on {endpoint}, creating another one...", endpoint)
|
log(f"Created container {cid} is not stored on {endpoint}, creating another one...", endpoint)
|
||||||
return create_container(endpoint, policy, container_creation_retry, wallet_path, config, acl, local, retry + 1)
|
return create_container(endpoint, policy, wallet_path, config, acl, local, depth + 1)
|
||||||
|
|
||||||
|
|
||||||
def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_config):
|
def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_config):
|
||||||
|
|
|
@ -20,7 +20,6 @@ DEFAULT_POLICY = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument('--size', help='Upload objects size in kb')
|
parser.add_argument('--size', help='Upload objects size in kb')
|
||||||
parser.add_argument('--containers', help='Number of containers to create')
|
parser.add_argument('--containers', help='Number of containers to create')
|
||||||
parser.add_argument('--retry', default=20, help='Maximum number of retries to create a container')
|
|
||||||
parser.add_argument('--out', help='JSON file with output')
|
parser.add_argument('--out', help='JSON file with output')
|
||||||
parser.add_argument('--preload_obj', help='Number of pre-loaded objects')
|
parser.add_argument('--preload_obj', help='Number of pre-loaded objects')
|
||||||
parser.add_argument('--wallet', help='Wallet file path')
|
parser.add_argument('--wallet', help='Wallet file path')
|
||||||
|
@ -51,7 +50,6 @@ def main():
|
||||||
if not args.policy:
|
if not args.policy:
|
||||||
args.policy = [DEFAULT_POLICY]
|
args.policy = [DEFAULT_POLICY]
|
||||||
|
|
||||||
container_creation_retry = args.retry
|
|
||||||
wallet = args.wallet
|
wallet = args.wallet
|
||||||
wallet_config = args.config
|
wallet_config = args.config
|
||||||
workers = int(args.workers)
|
workers = int(args.workers)
|
||||||
|
@ -68,7 +66,7 @@ def main():
|
||||||
containers_count = int(args.containers)
|
containers_count = int(args.containers)
|
||||||
print(f"Create containers: {containers_count}")
|
print(f"Create containers: {containers_count}")
|
||||||
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
|
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
|
||||||
containers_runs = [executor.submit(create_container, endpoint, policy, container_creation_retry, wallet, wallet_config, args.acl, args.local)
|
containers_runs = [executor.submit(create_container, endpoint, policy, wallet, wallet_config, args.acl, args.local)
|
||||||
for _, endpoint, policy in
|
for _, endpoint, policy in
|
||||||
zip(range(containers_count), cycle(endpoints), cycle(args.policy))]
|
zip(range(containers_count), cycle(endpoints), cycle(args.policy))]
|
||||||
|
|
||||||
|
|
|
@ -132,10 +132,6 @@ export function setup() {
|
||||||
const start_timestamp = Date.now()
|
const start_timestamp = Date.now()
|
||||||
console.log(
|
console.log(
|
||||||
`Load started at: ${Date(start_timestamp).toString()}`)
|
`Load started at: ${Date(start_timestamp).toString()}`)
|
||||||
|
|
||||||
if (delete_vu_count > 0){
|
|
||||||
obj_to_delete_selector.sync.add(delete_vu_count)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function teardown(data) {
|
export function teardown(data) {
|
||||||
|
@ -208,8 +204,6 @@ export function obj_delete() {
|
||||||
const obj = obj_to_delete_selector.nextObject();
|
const obj = obj_to_delete_selector.nextObject();
|
||||||
if (!obj) {
|
if (!obj) {
|
||||||
if (obj_to_delete_exit_on_null) {
|
if (obj_to_delete_exit_on_null) {
|
||||||
obj_to_delete_selector.sync.done()
|
|
||||||
obj_to_delete_selector.sync.wait()
|
|
||||||
exec.test.abort("No more objects to select");
|
exec.test.abort("No more objects to select");
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
|
|
Loading…
Reference in a new issue