TrueCloudLab/frostfs-node
21
1
Fork 27
Code Issues 108 Pull requests 3 Projects Releases 1 Activity Actions

[#135] tracing: Add tracing to node gRPC endpoints

Browse source 
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-03-13 14:01:43 +03:00 committed by fyrchik
parent 72565a91ef
commit 5af9f58469
12 changed files with 125 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -15,6 +15,7 @@ Changelog for FrostFS Node
- Multiple configs support (#44)
- Parameters `nns-name` and `nns-zone` for command `frostfs-cli container create` (#37)
- Tree service now saves the last synchronization height which persists across restarts (#82)
- Add tracing support (#135)
### Changed
- Change `frostfs_node_engine_container_size` to counting sizes of logical objects

12
cmd/frostfs-cli/modules/tree/client.go
View file

@ -5,6 +5,7 @@ import (
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree"
@ -22,8 +23,15 @@ func _client(ctx context.Context) (tree.TreeServiceClient, error) {
return nil, err
}
opts := make([]grpc.DialOption, 1, 2)
opts[0] = grpc.WithBlock()
opts := []grpc.DialOption{
grpc.WithBlock(),
grpc.WithChainUnaryInterceptor(
tracing.NewGRPCUnaryClientInteceptor(),
),
grpc.WithChainStreamInterceptor(
tracing.NewGRPCStreamClientInterceptor(),
),
}
if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") {
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))

9
cmd/frostfs-node/config.go
View file

@ -16,6 +16,7 @@ import (
"time"
netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient"
contractsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/contracts"
@ -27,6 +28,7 @@ import (
nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node"
objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object"
replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator"
tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
@ -1055,6 +1057,13 @@ func (c *cfg) reloadConfig(ctx context.Context) {
}
components = append(components, dCmp{"logger", logPrm.Reload})
components = append(components, dCmp{"tracing", func() error {
updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg))
if updated {
c.log.Info("tracing configation updated")
}
return err
}})
if cmp, updated := metricsComponent(c); updated {
if cmp.enabled {
cmp.preReload = enableMetricsSvc

31
cmd/frostfs-node/config/tracing/config.go Normal file
View file

@ -0,0 +1,31 @@
package tracing
import (
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
"git.frostfs.info/TrueCloudLab/frostfs-node/misc"
)
const (
subsection = "tracing"
)
// ToTracingConfig extracts tracing config.
func ToTracingConfig(c *config.Config) *tracing.Config {
return &tracing.Config{
Enabled: config.BoolSafe(c.Sub(subsection), "enabled"),
Exporter: tracing.Exporter(config.StringSafe(c.Sub(subsection), "exporter")),
Endpoint: config.StringSafe(c.Sub(subsection), "endpoint"),
Service: "frostfs-node",
InstanceID: getInstanceIDOrDefault(c),
Version: misc.Version,
}
}
func getInstanceIDOrDefault(c *config.Config) string {
s := config.StringSlice(c.Sub("node"), "addresses")
if len(s) > 0 {
return s[0]
}
return ""
}

7
cmd/frostfs-node/grpc.go
View file

@ -7,6 +7,7 @@ import (
"net"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
grpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/grpc"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"go.uber.org/zap"
@ -19,6 +20,12 @@ func initGRPC(c *cfg) {
grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) {
serverOpts := []grpc.ServerOption{
grpc.MaxSendMsgSize(maxMsgSize),
grpc.ChainUnaryInterceptor(
tracing.NewGRPCUnaryServerInterceptor(),
),
grpc.ChainStreamInterceptor(
tracing.NewGRPCStreamServerInterceptor(),
),
}
tlsCfg := sc.TLS()

2
cmd/frostfs-node/main.go
View file

@ -87,6 +87,8 @@ func initApp(ctx context.Context, c *cfg) {
initAndLog(c, pprof.name, pprof.init)
initAndLog(c, metrics.name, metrics.init)
initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) })
initLocalStorage(c)
initAndLog(c, "storage engine", func(c *cfg) {

31
cmd/frostfs-node/tracing.go Normal file
View file

@ -0,0 +1,31 @@
package main
import (
"context"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing"
"go.uber.org/zap"
)
func initTracing(ctx context.Context, c *cfg) {
conf := tracingconfig.ToTracingConfig(c.appCfg)
_, err := tracing.Setup(ctx, *conf)
if err != nil {
c.log.Error("failed init tracing", zap.Error(err))
}
c.closers = append(c.closers, closer{
name: "tracing",
fn: func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
err := tracing.Shutdown(ctx) //cfg context cancels before close
if err != nil {
c.log.Error("failed shutdown tracing", zap.Error(err))
}
},
})
}

4
config/example/node.env
View file

@ -184,3 +184,7 @@ FROSTFS_STORAGE_SHARD_1_PILORAMA_MAX_BATCH_SIZE=100
FROSTFS_STORAGE_SHARD_1_GC_REMOVER_BATCH_SIZE=200
#### Sleep interval between data remover tacts
FROSTFS_STORAGE_SHARD_1_GC_REMOVER_SLEEP_INTERVAL=5m
FROSTFS_TRACING_ENABLED=true
FROSTFS_TRACING_ENDPOINT="localhost"
FROSTFS_TRACING_EXPORTER="otlp_grpc"

5
config/example/node.json
View file

@ -243,5 +243,10 @@
}
}
}
},
"tracing": {
"enabled": true,
"endpoint": "localhost:9090",
"exporter": "otlp_grpc"
}
}

6
config/example/node.yaml
View file

@ -214,3 +214,9 @@ storage:
path: tmp/1/blob/pilorama.db
no_sync: true # USE WITH CAUTION. Return to user before pages have been persisted.
perm: 0644 # permission to use for the database file and intermediate directories
tracing:
enabled: true
exporter: "otlp_grpc"
endpoint: "localhost"

12
pkg/services/tree/cache.go
View file

@ -8,6 +8,7 @@ import (
"sync"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
"github.com/hashicorp/golang-lru/v2/simplelru"
"google.golang.org/grpc"
@ -84,8 +85,15 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn,
return nil, err
}
opts := make([]grpc.DialOption, 1, 2)
opts[0] = grpc.WithBlock()
opts := []grpc.DialOption{
grpc.WithBlock(),
grpc.WithChainUnaryInterceptor(
tracing.NewGRPCUnaryClientInteceptor(),
),
grpc.WithChainStreamInterceptor(
tracing.NewGRPCStreamClientInterceptor(),
),
}
// FIXME(@fyrchik): ugly hack #1322
if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") {

10
pkg/services/tree/sync.go
View file

@ -10,6 +10,7 @@ import (
"math/rand"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
@ -146,7 +147,14 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64,
return false
}
cc, err := grpc.DialContext(egCtx, a.URIAddr(), grpc.WithTransportCredentials(insecure.NewCredentials()))
cc, err := grpc.DialContext(egCtx, a.URIAddr(),
grpc.WithChainUnaryInterceptor(
tracing.NewGRPCUnaryClientInteceptor(),
),
grpc.WithChainStreamInterceptor(
tracing.NewGRPCStreamClientInterceptor(),
),
grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
// Failed to connect, try the next address.
return false