Compare commits
No commits in common. "master" and "feature-multi-load-policy" have entirely different histories.
master
...
feature-mu
10 changed files with 10 additions and 154 deletions
|
@ -3,8 +3,8 @@
|
|||
First, thank you for contributing! We love and encourage pull requests from
|
||||
everyone. Please follow the guidelines:
|
||||
|
||||
- Check the open [issues](https://git.frostfs.info/TrueCloudLab/xk6-frostfs/issues) and
|
||||
[pull requests](https://git.frostfs.info/TrueCloudLab/xk6-frostfs/pulls) for existing
|
||||
- Check the open [issues](https://github.com/TrueCloudLab/xk6-frostfs/issues) and
|
||||
[pull requests](https://github.com/TrueCloudLab/xk6-frostfs/pulls) for existing
|
||||
discussions.
|
||||
|
||||
- Open an issue first, to discuss a new feature or enhancement.
|
||||
|
@ -28,18 +28,18 @@ send a pull request. We encourage pull requests to discuss code changes. Here
|
|||
are the steps in details:
|
||||
|
||||
### Set up your GitHub Repository
|
||||
Fork [xk6-frostfs upstream](https://git.frostfs.info/TrueCloudLab/xk6-frostfs/fork) source
|
||||
Fork [xk6-frostfs upstream](https://github.com/TrueCloudLab/xk6-frostfs/fork) source
|
||||
repository to your own personal repository. Copy the URL of your fork (you will
|
||||
need it for the `git clone` command below).
|
||||
|
||||
```sh
|
||||
$ git clone https://git.frostfs.info/TrueCloudLab/xk6-frostfs
|
||||
$ git clone https://github.com/TrueCloudLab/xk6-frostfs
|
||||
```
|
||||
|
||||
### Set up git remote as ``upstream``
|
||||
```sh
|
||||
$ cd xk6-frostfs
|
||||
$ git remote add upstream https://git.frostfs.info/TrueCloudLab/xk6-frostfs
|
||||
$ git remote add upstream https://github.com/TrueCloudLab/xk6-frostfs
|
||||
$ git fetch upstream
|
||||
$ git merge upstream/master
|
||||
...
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
<p align="center">
|
||||
<img src="./.forgejo/logo.svg" width="500px" alt="FrostFS logo">
|
||||
<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://go.k6.io/k6">k6</a> extension to test and benchmark FrostFS related protocols.
|
||||
|
|
|
@ -1,55 +0,0 @@
|
|||
package importer
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/registry"
|
||||
)
|
||||
|
||||
type PreGenObj struct {
|
||||
Bucket string `json:"bucket"`
|
||||
Object string `json:"object"`
|
||||
Container string `json:"container"`
|
||||
}
|
||||
|
||||
type PreGenerateInfo struct {
|
||||
Buckets []string `json:"buckets"`
|
||||
Containers []string `json:"containers"`
|
||||
Objects []PreGenObj `json:"objects"`
|
||||
ObjSize string `json:"obj_size"`
|
||||
}
|
||||
|
||||
// ImportJSONPreGen writes objects from pregenerated JSON file
|
||||
// to the registry.
|
||||
// Note that ImportJSONPreGen does not check if object already
|
||||
// exists in the registry so in case of re-entry the registry
|
||||
// will have two entities representing the same object.
|
||||
func ImportJSONPreGen(o *registry.ObjRegistry, filename string) error {
|
||||
f, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var pregenInfo PreGenerateInfo
|
||||
err = json.Unmarshal(f, &pregenInfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// AddObject uses DB.Batch to combine concurrent Batch calls
|
||||
// into a single Bolt transaction. DB.Batch is limited by
|
||||
// DB.MaxBatchDelay which may affect perfomance.
|
||||
for _, obj := range pregenInfo.Objects {
|
||||
if obj.Bucket != "" {
|
||||
err = o.AddObject("", "", obj.Bucket, obj.Object, "")
|
||||
} else {
|
||||
err = o.AddObject(obj.Container, obj.Object, "", "", "")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
|
@ -1,27 +0,0 @@
|
|||
package importer
|
||||
|
||||
import (
|
||||
"git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/registry"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Cmd represents the import command.
|
||||
var Cmd = &cobra.Command{
|
||||
Use: "import",
|
||||
Short: "Import objects into registry",
|
||||
Long: "Import objects into registry from pregenerated files",
|
||||
Example: `xk6-registry import registry.bolt preset.json
|
||||
xk6-registry import --status created registry.bolt preset.json another_preset.json`,
|
||||
RunE: runCmd,
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
}
|
||||
|
||||
func runCmd(cmd *cobra.Command, args []string) error {
|
||||
objRegistry := registry.NewObjRegistry(cmd.Context(), args[0])
|
||||
for i := 1; i < len(args); i++ {
|
||||
if err := ImportJSONPreGen(objRegistry, args[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
|
@ -1,18 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
|
||||
|
||||
if cmd, err := rootCmd.ExecuteContextC(ctx); err != nil {
|
||||
cmd.PrintErrln("Error:", err.Error())
|
||||
cmd.PrintErrf("Run '%v --help' for usage.\n", cmd.CommandPath())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
|
@ -1,33 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/xk6-frostfs/cmd/xk6-registry/importer"
|
||||
"git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/version"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "xk6-registry",
|
||||
Version: version.Version,
|
||||
Short: "Command Line Tool to work with Registry",
|
||||
Long: `Registry provides tools to work with object registry for xk6.
|
||||
It contains command for importing objects in registry from preset`,
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
Run: rootCmdRun,
|
||||
}
|
||||
|
||||
func init() {
|
||||
cobra.AddTemplateFunc("runtimeVersion", runtime.Version)
|
||||
rootCmd.SetVersionTemplate(`FrostFS xk6-registry
|
||||
{{printf "Version: %s" .Version }}
|
||||
GoVersion: {{ runtimeVersion }}
|
||||
`)
|
||||
rootCmd.AddCommand(importer.Cmd)
|
||||
}
|
||||
|
||||
func rootCmdRun(cmd *cobra.Command, _ []string) {
|
||||
_ = cmd.Usage()
|
||||
}
|
|
@ -3,7 +3,6 @@ package registry
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neo-go/pkg/io"
|
||||
|
@ -24,8 +23,6 @@ type ObjSelector struct {
|
|||
filter *ObjFilter
|
||||
cacheSize int
|
||||
kind SelectorKind
|
||||
// Sync synchronizes VU used for deletion.
|
||||
Sync sync.WaitGroup
|
||||
}
|
||||
|
||||
// objectSelectCache is the default maximum size of a batch to select from DB.
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
import re
|
||||
from helpers.cmd import execute_cmd, log
|
||||
|
||||
def create_container(endpoint, policy, container_creation_retry, wallet_path, config, acl, local=False, retry=0):
|
||||
if retry > int(container_creation_retry):
|
||||
def create_container(endpoint, policy, wallet_path, config, acl, local=False, depth=0):
|
||||
if depth > 20:
|
||||
raise ValueError(f"unable to create container: too many unsuccessful attempts")
|
||||
|
||||
if wallet_path:
|
||||
|
@ -88,7 +88,7 @@ def create_container(endpoint, policy, container_creation_retry, wallet_path, co
|
|||
return cid
|
||||
|
||||
log(f"Created container {cid} is not stored on {endpoint}, creating another one...", endpoint)
|
||||
return create_container(endpoint, policy, container_creation_retry, wallet_path, config, acl, local, retry + 1)
|
||||
return create_container(endpoint, policy, wallet_path, config, acl, local, depth + 1)
|
||||
|
||||
|
||||
def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_config):
|
||||
|
|
|
@ -20,7 +20,6 @@ DEFAULT_POLICY = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
|
|||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--size', help='Upload objects size in kb')
|
||||
parser.add_argument('--containers', help='Number of containers to create')
|
||||
parser.add_argument('--retry', default=20, help='Maximum number of retries to create a container')
|
||||
parser.add_argument('--out', help='JSON file with output')
|
||||
parser.add_argument('--preload_obj', help='Number of pre-loaded objects')
|
||||
parser.add_argument('--wallet', help='Wallet file path')
|
||||
|
@ -51,7 +50,6 @@ def main():
|
|||
if not args.policy:
|
||||
args.policy = [DEFAULT_POLICY]
|
||||
|
||||
container_creation_retry = args.retry
|
||||
wallet = args.wallet
|
||||
wallet_config = args.config
|
||||
workers = int(args.workers)
|
||||
|
@ -68,7 +66,7 @@ def main():
|
|||
containers_count = int(args.containers)
|
||||
print(f"Create containers: {containers_count}")
|
||||
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
|
||||
containers_runs = [executor.submit(create_container, endpoint, policy, container_creation_retry, wallet, wallet_config, args.acl, args.local)
|
||||
containers_runs = [executor.submit(create_container, endpoint, policy, wallet, wallet_config, args.acl, args.local)
|
||||
for _, endpoint, policy in
|
||||
zip(range(containers_count), cycle(endpoints), cycle(args.policy))]
|
||||
|
||||
|
|
|
@ -132,10 +132,6 @@ export function setup() {
|
|||
const start_timestamp = Date.now()
|
||||
console.log(
|
||||
`Load started at: ${Date(start_timestamp).toString()}`)
|
||||
|
||||
if (delete_vu_count > 0){
|
||||
obj_to_delete_selector.sync.add(delete_vu_count)
|
||||
}
|
||||
}
|
||||
|
||||
export function teardown(data) {
|
||||
|
@ -208,8 +204,6 @@ export function obj_delete() {
|
|||
const obj = obj_to_delete_selector.nextObject();
|
||||
if (!obj) {
|
||||
if (obj_to_delete_exit_on_null) {
|
||||
obj_to_delete_selector.sync.done()
|
||||
obj_to_delete_selector.sync.wait()
|
||||
exec.test.abort("No more objects to select");
|
||||
}
|
||||
return;
|
||||
|
|
Loading…
Reference in a new issue