From 5ab7cfed7c7a6401584bd101f12334c2ba18fb81 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Wed, 19 Apr 2023 10:44:45 +0300 Subject: [PATCH] [#52] Add debug_logger flag to local scenarios Signed-off-by: Alejandro Lopez --- README.md | 4 +++- examples/local.js | 2 +- internal/local/local.go | 26 ++++++++++++++++++-------- internal/s3local/local.go | 7 ++++--- scenarios/local.js | 3 ++- scenarios/run_scenarios.md | 4 ++++ scenarios/s3local.js | 4 +++- 7 files changed, 35 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index df62f95..832f66e 100644 --- a/README.md +++ b/README.md @@ -74,10 +74,11 @@ const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0) Create a local client with `connect` method. Arguments: - local path to frostfs storage node configuration file - hex encoded private key (empty value produces random key) +- whether to use the debug logger (warning: very verbose) ```js import local from 'k6/x/frostfs/local'; -const local_client = local.connect("/path/to/config.yaml", "") +const local_client = local.connect("/path/to/config.yaml", "", false) ``` ### Methods @@ -125,6 +126,7 @@ Create local s3 client with `connect` method. Arguments: * `hex_key`: private key to use as a hexadecimal string. A random one is created if none is provided. * `node_position`: position of this node in the node array if loading multiple nodes independently (default: 0). * `node_count`: number of nodes in the node array if loading multiple nodes independently (default: 1). + * `debug_logger`: whether to use the development logger instead of the default. Helpful for debugging (default: false). - bucket-container mapping, which is needed to resolve the container id for a given bucket name. Any bucket used by the client must have an entry here. diff --git a/examples/local.js b/examples/local.js index 766f2e2..7e3b49a 100644 --- a/examples/local.js +++ b/examples/local.js @@ -2,7 +2,7 @@ import local from 'k6/x/frostfs/local'; import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js'; const payload = open('../go.sum', 'b'); -const local_cli = local.connect("/path/to/config.yaml", "") +const local_cli = local.connect("/path/to/config.yaml", "", false) export const options = { stages: [ diff --git a/internal/local/local.go b/internal/local/local.go index 88945c6..7f34100 100644 --- a/internal/local/local.go +++ b/internal/local/local.go @@ -47,7 +47,7 @@ type RootModule struct { // Local represents an instance of the module for every VU. type Local struct { vu modules.VU - ResolveEngine func(context.Context, string) (*engine.StorageEngine, error) + ResolveEngine func(context.Context, string, bool) (*engine.StorageEngine, error) } // Ensure the interfaces are implemented correctly. @@ -70,7 +70,7 @@ func (r *RootModule) NewModuleInstance(vu modules.VU) modules.Instance { return NewLocalModuleInstance(vu, r.GetOrCreateEngine) } -func NewLocalModuleInstance(vu modules.VU, resolveEngine func(context.Context, string) (*engine.StorageEngine, error)) *Local { +func NewLocalModuleInstance(vu modules.VU, resolveEngine func(context.Context, string, bool) (*engine.StorageEngine, error)) *Local { return &Local{ vu: vu, ResolveEngine: resolveEngine, @@ -80,7 +80,7 @@ func NewLocalModuleInstance(vu modules.VU, resolveEngine func(context.Context, s // GetOrCreateEngine returns the current engine instance for the given configuration file, // creating a new one if none exists. Note that the identity of configuration files is their // file name for the purposes of test runs. -func (r *RootModule) GetOrCreateEngine(ctx context.Context, configFile string) (*engine.StorageEngine, error) { +func (r *RootModule) GetOrCreateEngine(ctx context.Context, configFile string, debug bool) (*engine.StorageEngine, error) { r.mu.Lock() defer r.mu.Unlock() @@ -92,7 +92,10 @@ func (r *RootModule) GetOrCreateEngine(ctx context.Context, configFile string) ( if r.ng == nil { r.configFile = configFile appCfg := config.New(config.Prm{}, config.WithConfigFile(configFile)) - ngOpts, shardOpts := storageEngineOptionsFromConfig(appCfg) + ngOpts, shardOpts, err := storageEngineOptionsFromConfig(appCfg, debug) + if err != nil { + return nil, fmt.Errorf("creating engine options from config: %v", err) + } r.ng = engine.New(ngOpts...) for i, opts := range shardOpts { if _, err := r.ng.AddShard(opts...); err != nil { @@ -120,8 +123,8 @@ func (s *Local) Exports() modules.Exports { func (s *Local) VU() modules.VU { return s.vu } -func (s *Local) Connect(configFile, hexKey string) (*Client, error) { - ng, err := s.ResolveEngine(s.VU().Context(), configFile) +func (s *Local) Connect(configFile, hexKey string, debug bool) (*Client, error) { + ng, err := s.ResolveEngine(s.VU().Context(), configFile, debug) if err != nil { return nil, fmt.Errorf("connecting to engine for config %q: %v", configFile, err) } @@ -188,8 +191,15 @@ func (epochState) CurrentEpoch() uint64 { return 0 } // preloaded the storage (if any), by using the same configuration file. // // Note that the configuration file only needs to contain the storage-specific sections. -func storageEngineOptionsFromConfig(c *config.Config) ([]engine.Option, [][]shard.Option) { +func storageEngineOptionsFromConfig(c *config.Config, debug bool) ([]engine.Option, [][]shard.Option, error) { log := zap.L() + if debug { + var err error + log, err = zap.NewDevelopment() + if err != nil { + return nil, nil, fmt.Errorf("creating development logger: %v", err) + } + } ngOpts := []engine.Option{ engine.WithErrorThreshold(engineconfig.ShardErrorThreshold(c)), @@ -319,7 +329,7 @@ func storageEngineOptionsFromConfig(c *config.Config) ([]engine.Option, [][]shar return nil }) - return ngOpts, shOpts + return ngOpts, shOpts, nil } // ParseOrCreateKey parses the provided key as a hex string or creates a fresh one if empty. diff --git a/internal/s3local/local.go b/internal/s3local/local.go index f0ba436..34a5885 100644 --- a/internal/s3local/local.go +++ b/internal/s3local/local.go @@ -63,11 +63,12 @@ func (s *Local) Connect(configFile string, params map[string]string, bucketMappi hexKey := fs.String("hex_key", "", "Private key to use as a hexadecimal string. A random one is created if none is provided") nodePosition := fs.Int("node_position", 0, "Position of this node in the node array if loading multiple nodes independently") nodeCount := fs.Int("node_count", 1, "Number of nodes in the node array if loading multiple nodes independently") + debugLogger := fs.Bool("debug_logger", false, "Whether to use the development logger instead of the default one for debugging purposes") { - args := make([]string, 0, 2*len(params)) + args := make([]string, 0, len(params)) for k, v := range params { - args = append(args, "-"+k, v) + args = append(args, fmt.Sprintf("-%s=%s", k, v)) } if err := fs.Parse(args); err != nil { return nil, fmt.Errorf("parsing parameters: %v", err) @@ -106,7 +107,7 @@ func (s *Local) Connect(configFile string, params map[string]string, bucketMappi objGetDuration, _ = registry.NewMetric("s3local_obj_get_duration", metrics.Trend, metrics.Time) // Create S3 layer backed by local storage engine and tree service. - ng, err := s.l.ResolveEngine(s.l.VU().Context(), configFile) + ng, err := s.l.ResolveEngine(s.l.VU().Context(), configFile, *debugLogger) if err != nil { return nil, fmt.Errorf("connecting to engine for config %q: %v", configFile, err) } diff --git a/scenarios/local.js b/scenarios/local.js index 2c695e3..01496a4 100644 --- a/scenarios/local.js +++ b/scenarios/local.js @@ -21,7 +21,8 @@ const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json"; const config_file = __ENV.CONFIG_FILE; -const local_client = local.connect(config_file, ''); +const debug_logger = (__ENV.DEBUG_LOGGER || 'false') == 'true'; +const local_client = local.connect(config_file, '', debug_logger); const log = logging.new().withField("config", config_file); const registry_enabled = !!__ENV.REGISTRY_FILE; diff --git a/scenarios/run_scenarios.md b/scenarios/run_scenarios.md index 9d6bdb9..c39739b 100644 --- a/scenarios/run_scenarios.md +++ b/scenarios/run_scenarios.md @@ -19,6 +19,10 @@ Scenarios `grpc.js`, `local.js`, `http.js` and `s3.js` support the following opt * `SLEEP_READ` - time interval (in seconds) between reading VU iterations. * `SELECTION_SIZE` - size of batch to select for deletion (default: 1000). +## Common options for the local scenarios: + + * `DEBUG_LOGGER` - uses a development logger for the local storage engine to aid debugging (default: false). + Examples of how to use these options are provided below for each scenario. ## gRPC diff --git a/scenarios/s3local.js b/scenarios/s3local.js index b227e1e..4038eed 100644 --- a/scenarios/s3local.js +++ b/scenarios/s3local.js @@ -36,7 +36,9 @@ const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json"; const config_file = __ENV.CONFIG_FILE; -const s3_client = s3local.connect(config_file, {}, bucket_mapping()); +const s3_client = s3local.connect(config_file, { + 'debug_logger': __ENV.DEBUG_LOGGER || 'false', +}, bucket_mapping()); const log = logging.new().withField("config", config_file); const registry_enabled = !!__ENV.REGISTRY_FILE;