From 604982de3ed5a57a48408e1103d2034ccebf03aa Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 23 Jan 2024 18:21:51 +0300 Subject: [PATCH] [#119] metrics: Allow to add custom tags Signed-off-by: Dmitrii Stepanov --- internal/local/local.go | 19 ++++++++-------- internal/native/native.go | 27 +++++++++++----------- internal/s3/s3.go | 26 ++++++++++----------- internal/s3local/local.go | 26 ++++++++++----------- internal/stats/stats.go | 46 ++++++++++++++++++++++++++++++++++++-- scenarios/grpc.js | 5 +++++ scenarios/grpc_car.js | 5 +++++ scenarios/http.js | 5 +++++ scenarios/local.js | 5 +++++ scenarios/run_scenarios.md | 1 + scenarios/s3.js | 5 +++++ scenarios/s3_car.js | 5 +++++ scenarios/s3_multipart.js | 5 +++++ scenarios/s3local.js | 5 +++++ scenarios/verify.js | 5 +++++ 15 files changed, 138 insertions(+), 52 deletions(-) diff --git a/internal/local/local.go b/internal/local/local.go index 788061d..3c679bd 100644 --- a/internal/local/local.go +++ b/internal/local/local.go @@ -173,18 +173,17 @@ func (s *Local) Connect(configFile, configDir, hexKey string, debug bool, maxSiz } // Register metrics. - registry := metrics.NewRegistry() - objPutTotal, _ = registry.NewMetric("local_obj_put_total", metrics.Counter) - objPutFails, _ = registry.NewMetric("local_obj_put_fails", metrics.Counter) - objPutDuration, _ = registry.NewMetric("local_obj_put_duration", metrics.Trend, metrics.Time) + objPutTotal, _ = stats.Registry.NewMetric("local_obj_put_total", metrics.Counter) + objPutFails, _ = stats.Registry.NewMetric("local_obj_put_fails", metrics.Counter) + objPutDuration, _ = stats.Registry.NewMetric("local_obj_put_duration", metrics.Trend, metrics.Time) - objGetTotal, _ = registry.NewMetric("local_obj_get_total", metrics.Counter) - objGetFails, _ = registry.NewMetric("local_obj_get_fails", metrics.Counter) - objGetDuration, _ = registry.NewMetric("local_obj_get_duration", metrics.Trend, metrics.Time) + objGetTotal, _ = stats.Registry.NewMetric("local_obj_get_total", metrics.Counter) + objGetFails, _ = stats.Registry.NewMetric("local_obj_get_fails", metrics.Counter) + objGetDuration, _ = stats.Registry.NewMetric("local_obj_get_duration", metrics.Trend, metrics.Time) - objDeleteTotal, _ = registry.NewMetric("local_obj_delete_total", metrics.Counter) - objDeleteFails, _ = registry.NewMetric("local_obj_delete_fails", metrics.Counter) - objDeleteDuration, _ = registry.NewMetric("local_obj_delete_duration", metrics.Trend, metrics.Time) + objDeleteTotal, _ = stats.Registry.NewMetric("local_obj_delete_total", metrics.Counter) + objDeleteFails, _ = stats.Registry.NewMetric("local_obj_delete_fails", metrics.Counter) + objDeleteDuration, _ = stats.Registry.NewMetric("local_obj_delete_duration", metrics.Trend, metrics.Time) // Create raw client backed by local storage engine. rc := rawclient.New(ng, diff --git a/internal/native/native.go b/internal/native/native.go index ca02bbf..e5b4802 100644 --- a/internal/native/native.go +++ b/internal/native/native.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" + "git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/stats" "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "go.k6.io/k6/js/modules" @@ -115,22 +116,22 @@ func (n *Native) Connect(endpoint, hexPrivateKey string, dialTimeout, streamTime tok.SetExp(exp) // register metrics - registry := metrics.NewRegistry() - objPutTotal, _ = registry.NewMetric("frostfs_obj_put_total", metrics.Counter) - objPutFails, _ = registry.NewMetric("frostfs_obj_put_fails", metrics.Counter) - objPutDuration, _ = registry.NewMetric("frostfs_obj_put_duration", metrics.Trend, metrics.Time) - objGetTotal, _ = registry.NewMetric("frostfs_obj_get_total", metrics.Counter) - objGetFails, _ = registry.NewMetric("frostfs_obj_get_fails", metrics.Counter) - objGetDuration, _ = registry.NewMetric("frostfs_obj_get_duration", metrics.Trend, metrics.Time) + objPutTotal, _ = stats.Registry.NewMetric("frostfs_obj_put_total", metrics.Counter) + objPutFails, _ = stats.Registry.NewMetric("frostfs_obj_put_fails", metrics.Counter) + objPutDuration, _ = stats.Registry.NewMetric("frostfs_obj_put_duration", metrics.Trend, metrics.Time) - objDeleteTotal, _ = registry.NewMetric("frostfs_obj_delete_total", metrics.Counter) - objDeleteFails, _ = registry.NewMetric("frostfs_obj_delete_fails", metrics.Counter) - objDeleteDuration, _ = registry.NewMetric("frostfs_obj_delete_duration", metrics.Trend, metrics.Time) + objGetTotal, _ = stats.Registry.NewMetric("frostfs_obj_get_total", metrics.Counter) + objGetFails, _ = stats.Registry.NewMetric("frostfs_obj_get_fails", metrics.Counter) + objGetDuration, _ = stats.Registry.NewMetric("frostfs_obj_get_duration", metrics.Trend, metrics.Time) - cnrPutTotal, _ = registry.NewMetric("frostfs_cnr_put_total", metrics.Counter) - cnrPutFails, _ = registry.NewMetric("frostfs_cnr_put_fails", metrics.Counter) - cnrPutDuration, _ = registry.NewMetric("frostfs_cnr_put_duration", metrics.Trend, metrics.Time) + objDeleteTotal, _ = stats.Registry.NewMetric("frostfs_obj_delete_total", metrics.Counter) + objDeleteFails, _ = stats.Registry.NewMetric("frostfs_obj_delete_fails", metrics.Counter) + objDeleteDuration, _ = stats.Registry.NewMetric("frostfs_obj_delete_duration", metrics.Trend, metrics.Time) + + cnrPutTotal, _ = stats.Registry.NewMetric("frostfs_cnr_put_total", metrics.Counter) + cnrPutFails, _ = stats.Registry.NewMetric("frostfs_cnr_put_fails", metrics.Counter) + cnrPutDuration, _ = stats.Registry.NewMetric("frostfs_cnr_put_duration", metrics.Trend, metrics.Time) return &Client{ vu: n.vu, diff --git a/internal/s3/s3.go b/internal/s3/s3.go index 965bd50..3f7ed5c 100644 --- a/internal/s3/s3.go +++ b/internal/s3/s3.go @@ -7,6 +7,7 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/stats" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -94,22 +95,21 @@ func (s *S3) Connect(endpoint string, params map[string]string) (*Client, error) }) // register metrics - registry := metrics.NewRegistry() - objPutTotal, _ = registry.NewMetric("aws_obj_put_total", metrics.Counter) - objPutFails, _ = registry.NewMetric("aws_obj_put_fails", metrics.Counter) - objPutDuration, _ = registry.NewMetric("aws_obj_put_duration", metrics.Trend, metrics.Time) + objPutTotal, _ = stats.Registry.NewMetric("aws_obj_put_total", metrics.Counter) + objPutFails, _ = stats.Registry.NewMetric("aws_obj_put_fails", metrics.Counter) + objPutDuration, _ = stats.Registry.NewMetric("aws_obj_put_duration", metrics.Trend, metrics.Time) - objGetTotal, _ = registry.NewMetric("aws_obj_get_total", metrics.Counter) - objGetFails, _ = registry.NewMetric("aws_obj_get_fails", metrics.Counter) - objGetDuration, _ = registry.NewMetric("aws_obj_get_duration", metrics.Trend, metrics.Time) + objGetTotal, _ = stats.Registry.NewMetric("aws_obj_get_total", metrics.Counter) + objGetFails, _ = stats.Registry.NewMetric("aws_obj_get_fails", metrics.Counter) + objGetDuration, _ = stats.Registry.NewMetric("aws_obj_get_duration", metrics.Trend, metrics.Time) - objDeleteTotal, _ = registry.NewMetric("aws_obj_delete_total", metrics.Counter) - objDeleteFails, _ = registry.NewMetric("aws_obj_delete_fails", metrics.Counter) - objDeleteDuration, _ = registry.NewMetric("aws_obj_delete_duration", metrics.Trend, metrics.Time) + objDeleteTotal, _ = stats.Registry.NewMetric("aws_obj_delete_total", metrics.Counter) + objDeleteFails, _ = stats.Registry.NewMetric("aws_obj_delete_fails", metrics.Counter) + objDeleteDuration, _ = stats.Registry.NewMetric("aws_obj_delete_duration", metrics.Trend, metrics.Time) - createBucketTotal, _ = registry.NewMetric("aws_create_bucket_total", metrics.Counter) - createBucketFails, _ = registry.NewMetric("aws_create_bucket_fails", metrics.Counter) - createBucketDuration, _ = registry.NewMetric("aws_create_bucket_duration", metrics.Trend, metrics.Time) + createBucketTotal, _ = stats.Registry.NewMetric("aws_create_bucket_total", metrics.Counter) + createBucketFails, _ = stats.Registry.NewMetric("aws_create_bucket_fails", metrics.Counter) + createBucketDuration, _ = stats.Registry.NewMetric("aws_create_bucket_duration", metrics.Trend, metrics.Time) return &Client{ vu: s.vu, diff --git a/internal/s3local/local.go b/internal/s3local/local.go index 3da7435..68c16a8 100644 --- a/internal/s3local/local.go +++ b/internal/s3local/local.go @@ -88,23 +88,21 @@ func (s *Local) Connect(configFile string, configDir string, params map[string]s } // Register metrics. - registry := metrics.NewRegistry() + internalObjPutTotal, _ = stats.Registry.NewMetric("s3local_internal_obj_put_total", metrics.Counter) + internalObjPutFails, _ = stats.Registry.NewMetric("s3local_internal_obj_put_fails", metrics.Counter) + internalObjPutDuration, _ = stats.Registry.NewMetric("s3local_internal_obj_put_duration", metrics.Trend, metrics.Time) - internalObjPutTotal, _ = registry.NewMetric("s3local_internal_obj_put_total", metrics.Counter) - internalObjPutFails, _ = registry.NewMetric("s3local_internal_obj_put_fails", metrics.Counter) - internalObjPutDuration, _ = registry.NewMetric("s3local_internal_obj_put_duration", metrics.Trend, metrics.Time) + internalObjGetTotal, _ = stats.Registry.NewMetric("s3local_internal_obj_get_total", metrics.Counter) + internalObjGetFails, _ = stats.Registry.NewMetric("s3local_internal_obj_get_fails", metrics.Counter) + internalObjGetDuration, _ = stats.Registry.NewMetric("s3local_internal_obj_get_duration", metrics.Trend, metrics.Time) - internalObjGetTotal, _ = registry.NewMetric("s3local_internal_obj_get_total", metrics.Counter) - internalObjGetFails, _ = registry.NewMetric("s3local_internal_obj_get_fails", metrics.Counter) - internalObjGetDuration, _ = registry.NewMetric("s3local_internal_obj_get_duration", metrics.Trend, metrics.Time) + objPutTotal, _ = stats.Registry.NewMetric("s3local_obj_put_total", metrics.Counter) + objPutFails, _ = stats.Registry.NewMetric("s3local_obj_put_fails", metrics.Counter) + objPutDuration, _ = stats.Registry.NewMetric("s3local_obj_put_duration", metrics.Trend, metrics.Time) - objPutTotal, _ = registry.NewMetric("s3local_obj_put_total", metrics.Counter) - objPutFails, _ = registry.NewMetric("s3local_obj_put_fails", metrics.Counter) - objPutDuration, _ = registry.NewMetric("s3local_obj_put_duration", metrics.Trend, metrics.Time) - - objGetTotal, _ = registry.NewMetric("s3local_obj_get_total", metrics.Counter) - objGetFails, _ = registry.NewMetric("s3local_obj_get_fails", metrics.Counter) - objGetDuration, _ = registry.NewMetric("s3local_obj_get_duration", metrics.Trend, metrics.Time) + objGetTotal, _ = stats.Registry.NewMetric("s3local_obj_get_total", metrics.Counter) + objGetFails, _ = stats.Registry.NewMetric("s3local_obj_get_fails", metrics.Counter) + objGetDuration, _ = stats.Registry.NewMetric("s3local_obj_get_duration", metrics.Trend, metrics.Time) // Create S3 layer backed by local storage engine and tree service. ng, limiter, err := s.l.ResolveEngine(s.l.VU().Context(), configFile, configDir, *debugLogger, maxSizeGB) diff --git a/internal/stats/stats.go b/internal/stats/stats.go index 595e473..cf1a954 100644 --- a/internal/stats/stats.go +++ b/internal/stats/stats.go @@ -1,16 +1,54 @@ package stats import ( + "strings" "time" "go.k6.io/k6/js/modules" "go.k6.io/k6/metrics" ) +// RootModule is the global module object type. It is instantiated once per test +// run and will be used to create k6/x/frostfs/stats module instances for each VU. +type RootModule struct { + Instance string +} + +var ( + tagSet *metrics.TagSet + + Registry *metrics.Registry +) + +func init() { + Registry = metrics.NewRegistry() + tagSet = Registry.RootTagSet() + modules.Register("k6/x/frostfs/stats", &RootModule{}) +} + +// SetTags sets additional tags to custom metrics. +// Format: "key1:value1;key2:value2". +// Panics if input has invalid format. +func (m *RootModule) SetTags(labels string) { + kv := make(map[string]string) + pairs := strings.Split(labels, ";") + for _, pair := range pairs { + items := strings.Split(pair, ":") + if len(items) != 2 { + panic("invalid labels format") + } + kv[strings.TrimSpace(items[0])] = strings.TrimSpace(items[1]) + } + for k, v := range kv { + tagSet = tagSet.With(k, v) + } +} + func Report(vu modules.VU, metric *metrics.Metric, value float64) { metrics.PushIfNotDone(vu.Context(), vu.State().Samples, metrics.Sample{ TimeSeries: metrics.TimeSeries{ Metric: metric, + Tags: tagSet, }, Time: time.Now(), Value: value, @@ -22,9 +60,11 @@ func ReportDataReceived(vu modules.VU, value float64) { metrics.Sample{ TimeSeries: metrics.TimeSeries{ Metric: &metrics.Metric{}, + Tags: tagSet, }, Value: value, - Time: time.Now()}, + Time: time.Now(), + }, ) } @@ -34,8 +74,10 @@ func ReportDataSent(vu modules.VU, value float64) { metrics.Sample{ TimeSeries: metrics.TimeSeries{ Metric: &metrics.Metric{}, + Tags: tagSet, }, Value: value, - Time: time.Now()}, + Time: time.Now(), + }, ) } diff --git a/scenarios/grpc.js b/scenarios/grpc.js index 6f675cd..65d11d9 100644 --- a/scenarios/grpc.js +++ b/scenarios/grpc.js @@ -1,6 +1,7 @@ import native from 'k6/x/frostfs/native'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; +import stats from 'k6/x/frostfs/stats'; import { SharedArray } from 'k6/data'; import { sleep } from 'k6'; import { textSummary } from './libs/k6-summary-0.0.2.js'; @@ -35,6 +36,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und const duration = __ENV.DURATION; +if (!!__ENV.METRIC_TAGS) { + stats.setTags(__ENV.METRIC_TAGS) +} + const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; let obj_to_delete_selector = undefined; if (registry_enabled && delete_age) { diff --git a/scenarios/grpc_car.js b/scenarios/grpc_car.js index 19ceaf9..c69ee7e 100644 --- a/scenarios/grpc_car.js +++ b/scenarios/grpc_car.js @@ -1,6 +1,7 @@ import native from 'k6/x/frostfs/native'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; +import stats from 'k6/x/frostfs/stats'; import { SharedArray } from 'k6/data'; import { sleep } from 'k6'; import { textSummary } from './libs/k6-summary-0.0.2.js'; @@ -35,6 +36,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und const duration = __ENV.DURATION; +if (!!__ENV.METRIC_TAGS) { + stats.setTags(__ENV.METRIC_TAGS) +} + const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; let obj_to_delete_selector = undefined; if (registry_enabled && delete_age) { diff --git a/scenarios/http.js b/scenarios/http.js index 81215e6..f96c0af 100644 --- a/scenarios/http.js +++ b/scenarios/http.js @@ -1,5 +1,6 @@ import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; +import stats from 'k6/x/frostfs/stats'; import http from 'k6/http'; import { SharedArray } from 'k6/data'; import { sleep } from 'k6'; @@ -31,6 +32,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und const duration = __ENV.DURATION; +if (!!__ENV.METRIC_TAGS) { + stats.setTags(__ENV.METRIC_TAGS) +} + const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); diff --git a/scenarios/local.js b/scenarios/local.js index c90610c..0630304 100644 --- a/scenarios/local.js +++ b/scenarios/local.js @@ -1,6 +1,7 @@ import local from 'k6/x/frostfs/local'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; +import stats from 'k6/x/frostfs/stats'; import { SharedArray } from 'k6/data'; import { textSummary } from './libs/k6-summary-0.0.2.js'; import { parseEnv } from './libs/env-parser.js'; @@ -33,6 +34,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und const duration = __ENV.DURATION; +if (!!__ENV.METRIC_TAGS) { + stats.setTags(__ENV.METRIC_TAGS) +} + const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; let obj_to_delete_selector = undefined; if (registry_enabled && delete_age) { diff --git a/scenarios/run_scenarios.md b/scenarios/run_scenarios.md index c94ad5a..e3d38e5 100644 --- a/scenarios/run_scenarios.md +++ b/scenarios/run_scenarios.md @@ -20,6 +20,7 @@ Scenarios `grpc.js`, `local.js`, `http.js` and `s3.js` support the following opt * `SELECTION_SIZE` - size of batch to select for deletion (default: 1000). * `PAYLOAD_TYPE` - type of an object payload ("random" or "text", default: "random"). * `STREAMING` - if set, the payload is generated on the fly and is not read into memory fully. + * `METRIC_TAGS` - `instance` tag value. Additionally, the profiling extension can be enabled to generate CPU and memory profiles which can be inspected with `go tool pprof file.prof`: ```shell diff --git a/scenarios/s3.js b/scenarios/s3.js index 814f073..51eed76 100644 --- a/scenarios/s3.js +++ b/scenarios/s3.js @@ -1,5 +1,6 @@ import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; +import stats from 'k6/x/frostfs/stats'; import s3 from 'k6/x/frostfs/s3'; import { SharedArray } from 'k6/data'; import { sleep } from 'k6'; @@ -34,6 +35,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und const duration = __ENV.DURATION; +if (!!__ENV.METRIC_TAGS) { + stats.setTags(__ENV.METRIC_TAGS) +} + const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; let obj_to_delete_selector = undefined; if (registry_enabled && delete_age) { diff --git a/scenarios/s3_car.js b/scenarios/s3_car.js index 6eb94c6..74eba7a 100644 --- a/scenarios/s3_car.js +++ b/scenarios/s3_car.js @@ -1,5 +1,6 @@ import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; +import stats from 'k6/x/frostfs/stats'; import s3 from 'k6/x/frostfs/s3'; import { SharedArray } from 'k6/data'; import { sleep } from 'k6'; @@ -34,6 +35,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und const duration = __ENV.DURATION; +if (!!__ENV.METRIC_TAGS) { + stats.setTags(__ENV.METRIC_TAGS) +} + const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; let obj_to_delete_selector = undefined; if (registry_enabled && delete_age) { diff --git a/scenarios/s3_multipart.js b/scenarios/s3_multipart.js index 43e0607..a3c1e28 100644 --- a/scenarios/s3_multipart.js +++ b/scenarios/s3_multipart.js @@ -1,5 +1,6 @@ import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; +import stats from 'k6/x/frostfs/stats'; import s3 from 'k6/x/frostfs/s3'; import {SharedArray} from 'k6/data'; import {sleep} from 'k6'; @@ -29,6 +30,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und const duration = __ENV.DURATION; +if (!!__ENV.METRIC_TAGS) { + stats.setTags(__ENV.METRIC_TAGS) +} + const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); diff --git a/scenarios/s3local.js b/scenarios/s3local.js index fe31ed0..17dee46 100644 --- a/scenarios/s3local.js +++ b/scenarios/s3local.js @@ -1,5 +1,6 @@ import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; +import stats from 'k6/x/frostfs/stats'; import s3local from 'k6/x/frostfs/s3local'; import { SharedArray } from 'k6/data'; import { textSummary } from './libs/k6-summary-0.0.2.js'; @@ -44,6 +45,10 @@ const s3_client = s3local.connect(config_file, config_dir, { }, bucket_mapping(), max_total_size_gb); const log = logging.new().withFields({"config_file": config_file,"config_dir": config_dir}); +if (!!__ENV.METRIC_TAGS) { + stats.setTags(__ENV.METRIC_TAGS) +} + const registry_enabled = !!__ENV.REGISTRY_FILE; const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; diff --git a/scenarios/verify.js b/scenarios/verify.js index da27fed..44318c6 100644 --- a/scenarios/verify.js +++ b/scenarios/verify.js @@ -1,5 +1,6 @@ import native from 'k6/x/frostfs/native'; import registry from 'k6/x/frostfs/registry'; +import stats from 'k6/x/frostfs/stats'; import s3 from 'k6/x/frostfs/s3'; import logging from 'k6/x/frostfs/logging'; import { sleep } from 'k6'; @@ -28,6 +29,10 @@ const obj_counters = { let log = logging.new(); +if (!!__ENV.METRIC_TAGS) { + stats.setTags(__ENV.METRIC_TAGS) +} + // Connect to random gRPC endpoint let grpc_client = undefined; if (__ENV.GRPC_ENDPOINTS) {