[#135] tracing: Add tracing to node gRPC endpoints

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

CHANGELOG.md

@@ -15,6 +15,7 @@ Changelog for FrostFS Node
 - Multiple configs support (#44)
 - Parameters `nns-name` and `nns-zone` for command `frostfs-cli container create` (#37)
 - Tree service now saves the last synchronization height which persists across restarts (#82)
+- Add tracing support (#135)
 
 ### Changed
 - Change `frostfs_node_engine_container_size` to counting sizes of logical objects

cmd/frostfs-cli/modules/tree/client.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"time"
 
+	"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree"
@@ -22,8 +23,15 @@ func _client(ctx context.Context) (tree.TreeServiceClient, error) {
 		return nil, err
 	}
 
-	opts := make([]grpc.DialOption, 1, 2)
+	opts := []grpc.DialOption{
-	opts[0] = grpc.WithBlock()
+		grpc.WithBlock(),
+		grpc.WithChainUnaryInterceptor(
+			tracing.NewGRPCUnaryClientInteceptor(),
+		),
+		grpc.WithChainStreamInterceptor(
+			tracing.NewGRPCStreamClientInterceptor(),
+		),
+	}
 
 	if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") {
 		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))

cmd/frostfs-node/config.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
+	"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
 	apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient"
 	contractsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/contracts"
@@ -27,6 +28,7 @@ import (
 	nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node"
 	objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object"
 	replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator"
+	tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
 	netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
@@ -1055,6 +1057,13 @@ func (c *cfg) reloadConfig(ctx context.Context) {
 	}
 
 	components = append(components, dCmp{"logger", logPrm.Reload})
+	components = append(components, dCmp{"tracing", func() error {
+		updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg))
+		if updated {
+			c.log.Info("tracing configation updated")
+		}
+		return err
+	}})
 	if cmp, updated := metricsComponent(c); updated {
 		if cmp.enabled {
 			cmp.preReload = enableMetricsSvc

cmd/frostfs-node/config/tracing/config.go

@@ -0,0 +1,31 @@
+package tracing
+
+import (
+	"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/misc"
+)
+
+const (
+	subsection = "tracing"
+)
+
+// ToTracingConfig extracts tracing config.
+func ToTracingConfig(c *config.Config) *tracing.Config {
+	return &tracing.Config{
+		Enabled:    config.BoolSafe(c.Sub(subsection), "enabled"),
+		Exporter:   tracing.Exporter(config.StringSafe(c.Sub(subsection), "exporter")),
+		Endpoint:   config.StringSafe(c.Sub(subsection), "endpoint"),
+		Service:    "frostfs-node",
+		InstanceID: getInstanceIDOrDefault(c),
+		Version:    misc.Version,
+	}
+}
+
+func getInstanceIDOrDefault(c *config.Config) string {
+	s := config.StringSlice(c.Sub("node"), "addresses")
+	if len(s) > 0 {
+		return s[0]
+	}
+	return ""
+}

cmd/frostfs-node/grpc.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"time"
 
+	"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
 	grpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/grpc"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
 	"go.uber.org/zap"
@@ -19,6 +20,12 @@ func initGRPC(c *cfg) {
 	grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) {
 		serverOpts := []grpc.ServerOption{
 			grpc.MaxSendMsgSize(maxMsgSize),
+			grpc.ChainUnaryInterceptor(
+				tracing.NewGRPCUnaryServerInterceptor(),
+			),
+			grpc.ChainStreamInterceptor(
+				tracing.NewGRPCStreamServerInterceptor(),
+			),
 		}
 
 		tlsCfg := sc.TLS()

cmd/frostfs-node/main.go

@@ -87,6 +87,8 @@ func initApp(ctx context.Context, c *cfg) {
 	initAndLog(c, pprof.name, pprof.init)
 	initAndLog(c, metrics.name, metrics.init)
 
+	initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) })
+
 	initLocalStorage(c)
 
 	initAndLog(c, "storage engine", func(c *cfg) {

cmd/frostfs-node/tracing.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"context"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
+	tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing"
+	"go.uber.org/zap"
+)
+
+func initTracing(ctx context.Context, c *cfg) {
+	conf := tracingconfig.ToTracingConfig(c.appCfg)
+
+	_, err := tracing.Setup(ctx, *conf)
+	if err != nil {
+		c.log.Error("failed init tracing", zap.Error(err))
+	}
+
+	c.closers = append(c.closers, closer{
+		name: "tracing",
+		fn: func() {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+			defer cancel()
+			err := tracing.Shutdown(ctx) //cfg context cancels before close
+			if err != nil {
+				c.log.Error("failed shutdown tracing", zap.Error(err))
+			}
+		},
+	})
+}

config/example/node.env

@@ -184,3 +184,7 @@ FROSTFS_STORAGE_SHARD_1_PILORAMA_MAX_BATCH_SIZE=100
 FROSTFS_STORAGE_SHARD_1_GC_REMOVER_BATCH_SIZE=200
 #### Sleep interval between data remover tacts
 FROSTFS_STORAGE_SHARD_1_GC_REMOVER_SLEEP_INTERVAL=5m
+
+FROSTFS_TRACING_ENABLED=true
+FROSTFS_TRACING_ENDPOINT="localhost"
+FROSTFS_TRACING_EXPORTER="otlp_grpc"

config/example/node.json

@@ -243,5 +243,10 @@
         }
       }
     }
+  },
+  "tracing": {
+    "enabled": true,
+    "endpoint": "localhost:9090",
+    "exporter": "otlp_grpc"
   }
 }

config/example/node.yaml

@@ -214,3 +214,9 @@ storage:
         path: tmp/1/blob/pilorama.db
         no_sync: true # USE WITH CAUTION. Return to user before pages have been persisted.
         perm: 0644 # permission to use for the database file and intermediate directories
+
+tracing:
+  enabled: true
+  exporter: "otlp_grpc"
+  endpoint: "localhost"
+  

pkg/services/tree/cache.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"time"
 
+	"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
 	"github.com/hashicorp/golang-lru/v2/simplelru"
 	"google.golang.org/grpc"
@@ -84,8 +85,15 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn,
 		return nil, err
 	}
 
-	opts := make([]grpc.DialOption, 1, 2)
+	opts := []grpc.DialOption{
-	opts[0] = grpc.WithBlock()
+		grpc.WithBlock(),
+		grpc.WithChainUnaryInterceptor(
+			tracing.NewGRPCUnaryClientInteceptor(),
+		),
+		grpc.WithChainStreamInterceptor(
+			tracing.NewGRPCStreamClientInterceptor(),
+		),
+	}
 
 	// FIXME(@fyrchik): ugly hack #1322
 	if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") {

pkg/services/tree/sync.go

@@ -10,6 +10,7 @@ import (
 	"math/rand"
 	"sync"
 
+	"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
@@ -146,7 +147,14 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64,
 					return false
 				}
 
-				cc, err := grpc.DialContext(egCtx, a.URIAddr(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+				cc, err := grpc.DialContext(egCtx, a.URIAddr(),
+					grpc.WithChainUnaryInterceptor(
+						tracing.NewGRPCUnaryClientInteceptor(),
+					),
+					grpc.WithChainStreamInterceptor(
+						tracing.NewGRPCStreamClientInterceptor(),
+					),
+					grpc.WithTransportCredentials(insecure.NewCredentials()))
 				if err != nil {
 					// Failed to connect, try the next address.
 					return false