From 5af9f5846981b7262ce941c417e470059ada8937 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 13 Mar 2023 14:01:43 +0300 Subject: [PATCH] [#135] tracing: Add tracing to node gRPC endpoints Signed-off-by: Dmitrii Stepanov --- CHANGELOG.md | 1 + cmd/frostfs-cli/modules/tree/client.go | 12 +++++++-- cmd/frostfs-node/config.go | 9 +++++++ cmd/frostfs-node/config/tracing/config.go | 31 +++++++++++++++++++++++ cmd/frostfs-node/grpc.go | 7 +++++ cmd/frostfs-node/main.go | 2 ++ cmd/frostfs-node/tracing.go | 31 +++++++++++++++++++++++ config/example/node.env | 4 +++ config/example/node.json | 5 ++++ config/example/node.yaml | 6 +++++ pkg/services/tree/cache.go | 12 +++++++-- pkg/services/tree/sync.go | 10 +++++++- 12 files changed, 125 insertions(+), 5 deletions(-) create mode 100644 cmd/frostfs-node/config/tracing/config.go create mode 100644 cmd/frostfs-node/tracing.go diff --git a/CHANGELOG.md b/CHANGELOG.md index db846936f..0e07bb2f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Changelog for FrostFS Node - Multiple configs support (#44) - Parameters `nns-name` and `nns-zone` for command `frostfs-cli container create` (#37) - Tree service now saves the last synchronization height which persists across restarts (#82) +- Add tracing support (#135) ### Changed - Change `frostfs_node_engine_container_size` to counting sizes of logical objects diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index f379de41b..f25bff166 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" @@ -22,8 +23,15 @@ func _client(ctx context.Context) (tree.TreeServiceClient, error) { return nil, err } - opts := make([]grpc.DialOption, 1, 2) - opts[0] = grpc.WithBlock() + opts := []grpc.DialOption{ + grpc.WithBlock(), + grpc.WithChainUnaryInterceptor( + tracing.NewGRPCUnaryClientInteceptor(), + ), + grpc.WithChainStreamInterceptor( + tracing.NewGRPCStreamClientInterceptor(), + ), + } if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index ab615d340..d81e47b17 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -16,6 +16,7 @@ import ( "time" netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient" contractsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/contracts" @@ -27,6 +28,7 @@ import ( nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" + tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -1055,6 +1057,13 @@ func (c *cfg) reloadConfig(ctx context.Context) { } components = append(components, dCmp{"logger", logPrm.Reload}) + components = append(components, dCmp{"tracing", func() error { + updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg)) + if updated { + c.log.Info("tracing configation updated") + } + return err + }}) if cmp, updated := metricsComponent(c); updated { if cmp.enabled { cmp.preReload = enableMetricsSvc diff --git a/cmd/frostfs-node/config/tracing/config.go b/cmd/frostfs-node/config/tracing/config.go new file mode 100644 index 000000000..76572cc31 --- /dev/null +++ b/cmd/frostfs-node/config/tracing/config.go @@ -0,0 +1,31 @@ +package tracing + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/misc" +) + +const ( + subsection = "tracing" +) + +// ToTracingConfig extracts tracing config. +func ToTracingConfig(c *config.Config) *tracing.Config { + return &tracing.Config{ + Enabled: config.BoolSafe(c.Sub(subsection), "enabled"), + Exporter: tracing.Exporter(config.StringSafe(c.Sub(subsection), "exporter")), + Endpoint: config.StringSafe(c.Sub(subsection), "endpoint"), + Service: "frostfs-node", + InstanceID: getInstanceIDOrDefault(c), + Version: misc.Version, + } +} + +func getInstanceIDOrDefault(c *config.Config) string { + s := config.StringSlice(c.Sub("node"), "addresses") + if len(s) > 0 { + return s[0] + } + return "" +} diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index a56c76606..f3943f3ff 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -7,6 +7,7 @@ import ( "net" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" grpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" @@ -19,6 +20,12 @@ func initGRPC(c *cfg) { grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) { serverOpts := []grpc.ServerOption{ grpc.MaxSendMsgSize(maxMsgSize), + grpc.ChainUnaryInterceptor( + tracing.NewGRPCUnaryServerInterceptor(), + ), + grpc.ChainStreamInterceptor( + tracing.NewGRPCStreamServerInterceptor(), + ), } tlsCfg := sc.TLS() diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index cddedabe9..2f4c9853f 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -87,6 +87,8 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(c, pprof.name, pprof.init) initAndLog(c, metrics.name, metrics.init) + initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) }) + initLocalStorage(c) initAndLog(c, "storage engine", func(c *cfg) { diff --git a/cmd/frostfs-node/tracing.go b/cmd/frostfs-node/tracing.go new file mode 100644 index 000000000..bbdb71c64 --- /dev/null +++ b/cmd/frostfs-node/tracing.go @@ -0,0 +1,31 @@ +package main + +import ( + "context" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" + "go.uber.org/zap" +) + +func initTracing(ctx context.Context, c *cfg) { + conf := tracingconfig.ToTracingConfig(c.appCfg) + + _, err := tracing.Setup(ctx, *conf) + if err != nil { + c.log.Error("failed init tracing", zap.Error(err)) + } + + c.closers = append(c.closers, closer{ + name: "tracing", + fn: func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + err := tracing.Shutdown(ctx) //cfg context cancels before close + if err != nil { + c.log.Error("failed shutdown tracing", zap.Error(err)) + } + }, + }) +} diff --git a/config/example/node.env b/config/example/node.env index 8034fbb23..9a1a8b052 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -184,3 +184,7 @@ FROSTFS_STORAGE_SHARD_1_PILORAMA_MAX_BATCH_SIZE=100 FROSTFS_STORAGE_SHARD_1_GC_REMOVER_BATCH_SIZE=200 #### Sleep interval between data remover tacts FROSTFS_STORAGE_SHARD_1_GC_REMOVER_SLEEP_INTERVAL=5m + +FROSTFS_TRACING_ENABLED=true +FROSTFS_TRACING_ENDPOINT="localhost" +FROSTFS_TRACING_EXPORTER="otlp_grpc" \ No newline at end of file diff --git a/config/example/node.json b/config/example/node.json index e7bb375a5..8cfb5bb69 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -243,5 +243,10 @@ } } } + }, + "tracing": { + "enabled": true, + "endpoint": "localhost:9090", + "exporter": "otlp_grpc" } } diff --git a/config/example/node.yaml b/config/example/node.yaml index 6a5ea5f03..e3b41d413 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -214,3 +214,9 @@ storage: path: tmp/1/blob/pilorama.db no_sync: true # USE WITH CAUTION. Return to user before pages have been persisted. perm: 0644 # permission to use for the database file and intermediate directories + +tracing: + enabled: true + exporter: "otlp_grpc" + endpoint: "localhost" + \ No newline at end of file diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 73745e1b1..ab9f509ac 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "github.com/hashicorp/golang-lru/v2/simplelru" "google.golang.org/grpc" @@ -84,8 +85,15 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, return nil, err } - opts := make([]grpc.DialOption, 1, 2) - opts[0] = grpc.WithBlock() + opts := []grpc.DialOption{ + grpc.WithBlock(), + grpc.WithChainUnaryInterceptor( + tracing.NewGRPCUnaryClientInteceptor(), + ), + grpc.WithChainStreamInterceptor( + tracing.NewGRPCStreamClientInterceptor(), + ), + } // FIXME(@fyrchik): ugly hack #1322 if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") { diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 32d088c01..47299d1c9 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -10,6 +10,7 @@ import ( "math/rand" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" @@ -146,7 +147,14 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, return false } - cc, err := grpc.DialContext(egCtx, a.URIAddr(), grpc.WithTransportCredentials(insecure.NewCredentials())) + cc, err := grpc.DialContext(egCtx, a.URIAddr(), + grpc.WithChainUnaryInterceptor( + tracing.NewGRPCUnaryClientInteceptor(), + ), + grpc.WithChainStreamInterceptor( + tracing.NewGRPCStreamClientInterceptor(), + ), + grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { // Failed to connect, try the next address. return false