forked from TrueCloudLab/frostfs-node
[#135] tracing: Add tracing to node gRPC endpoints
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
parent
72565a91ef
commit
5af9f58469
12 changed files with 125 additions and 5 deletions
|
@ -15,6 +15,7 @@ Changelog for FrostFS Node
|
|||
- Multiple configs support (#44)
|
||||
- Parameters `nns-name` and `nns-zone` for command `frostfs-cli container create` (#37)
|
||||
- Tree service now saves the last synchronization height which persists across restarts (#82)
|
||||
- Add tracing support (#135)
|
||||
|
||||
### Changed
|
||||
- Change `frostfs_node_engine_container_size` to counting sizes of logical objects
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree"
|
||||
|
@ -22,8 +23,15 @@ func _client(ctx context.Context) (tree.TreeServiceClient, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
opts := make([]grpc.DialOption, 1, 2)
|
||||
opts[0] = grpc.WithBlock()
|
||||
opts := []grpc.DialOption{
|
||||
grpc.WithBlock(),
|
||||
grpc.WithChainUnaryInterceptor(
|
||||
tracing.NewGRPCUnaryClientInteceptor(),
|
||||
),
|
||||
grpc.WithChainStreamInterceptor(
|
||||
tracing.NewGRPCStreamClientInterceptor(),
|
||||
),
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") {
|
||||
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
|
|
|
@ -16,6 +16,7 @@ import (
|
|||
"time"
|
||||
|
||||
netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
|
||||
apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient"
|
||||
contractsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/contracts"
|
||||
|
@ -27,6 +28,7 @@ import (
|
|||
nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node"
|
||||
objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object"
|
||||
replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator"
|
||||
tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
|
||||
netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
|
||||
|
@ -1055,6 +1057,13 @@ func (c *cfg) reloadConfig(ctx context.Context) {
|
|||
}
|
||||
|
||||
components = append(components, dCmp{"logger", logPrm.Reload})
|
||||
components = append(components, dCmp{"tracing", func() error {
|
||||
updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg))
|
||||
if updated {
|
||||
c.log.Info("tracing configation updated")
|
||||
}
|
||||
return err
|
||||
}})
|
||||
if cmp, updated := metricsComponent(c); updated {
|
||||
if cmp.enabled {
|
||||
cmp.preReload = enableMetricsSvc
|
||||
|
|
31
cmd/frostfs-node/config/tracing/config.go
Normal file
31
cmd/frostfs-node/config/tracing/config.go
Normal file
|
@ -0,0 +1,31 @@
|
|||
package tracing
|
||||
|
||||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/misc"
|
||||
)
|
||||
|
||||
const (
|
||||
subsection = "tracing"
|
||||
)
|
||||
|
||||
// ToTracingConfig extracts tracing config.
|
||||
func ToTracingConfig(c *config.Config) *tracing.Config {
|
||||
return &tracing.Config{
|
||||
Enabled: config.BoolSafe(c.Sub(subsection), "enabled"),
|
||||
Exporter: tracing.Exporter(config.StringSafe(c.Sub(subsection), "exporter")),
|
||||
Endpoint: config.StringSafe(c.Sub(subsection), "endpoint"),
|
||||
Service: "frostfs-node",
|
||||
InstanceID: getInstanceIDOrDefault(c),
|
||||
Version: misc.Version,
|
||||
}
|
||||
}
|
||||
|
||||
func getInstanceIDOrDefault(c *config.Config) string {
|
||||
s := config.StringSlice(c.Sub("node"), "addresses")
|
||||
if len(s) > 0 {
|
||||
return s[0]
|
||||
}
|
||||
return ""
|
||||
}
|
|
@ -7,6 +7,7 @@ import (
|
|||
"net"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
|
||||
grpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/grpc"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
|
||||
"go.uber.org/zap"
|
||||
|
@ -19,6 +20,12 @@ func initGRPC(c *cfg) {
|
|||
grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) {
|
||||
serverOpts := []grpc.ServerOption{
|
||||
grpc.MaxSendMsgSize(maxMsgSize),
|
||||
grpc.ChainUnaryInterceptor(
|
||||
tracing.NewGRPCUnaryServerInterceptor(),
|
||||
),
|
||||
grpc.ChainStreamInterceptor(
|
||||
tracing.NewGRPCStreamServerInterceptor(),
|
||||
),
|
||||
}
|
||||
|
||||
tlsCfg := sc.TLS()
|
||||
|
|
|
@ -87,6 +87,8 @@ func initApp(ctx context.Context, c *cfg) {
|
|||
initAndLog(c, pprof.name, pprof.init)
|
||||
initAndLog(c, metrics.name, metrics.init)
|
||||
|
||||
initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) })
|
||||
|
||||
initLocalStorage(c)
|
||||
|
||||
initAndLog(c, "storage engine", func(c *cfg) {
|
||||
|
|
31
cmd/frostfs-node/tracing.go
Normal file
31
cmd/frostfs-node/tracing.go
Normal file
|
@ -0,0 +1,31 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
|
||||
tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func initTracing(ctx context.Context, c *cfg) {
|
||||
conf := tracingconfig.ToTracingConfig(c.appCfg)
|
||||
|
||||
_, err := tracing.Setup(ctx, *conf)
|
||||
if err != nil {
|
||||
c.log.Error("failed init tracing", zap.Error(err))
|
||||
}
|
||||
|
||||
c.closers = append(c.closers, closer{
|
||||
name: "tracing",
|
||||
fn: func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
err := tracing.Shutdown(ctx) //cfg context cancels before close
|
||||
if err != nil {
|
||||
c.log.Error("failed shutdown tracing", zap.Error(err))
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
|
@ -184,3 +184,7 @@ FROSTFS_STORAGE_SHARD_1_PILORAMA_MAX_BATCH_SIZE=100
|
|||
FROSTFS_STORAGE_SHARD_1_GC_REMOVER_BATCH_SIZE=200
|
||||
#### Sleep interval between data remover tacts
|
||||
FROSTFS_STORAGE_SHARD_1_GC_REMOVER_SLEEP_INTERVAL=5m
|
||||
|
||||
FROSTFS_TRACING_ENABLED=true
|
||||
FROSTFS_TRACING_ENDPOINT="localhost"
|
||||
FROSTFS_TRACING_EXPORTER="otlp_grpc"
|
|
@ -243,5 +243,10 @@
|
|||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tracing": {
|
||||
"enabled": true,
|
||||
"endpoint": "localhost:9090",
|
||||
"exporter": "otlp_grpc"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -214,3 +214,9 @@ storage:
|
|||
path: tmp/1/blob/pilorama.db
|
||||
no_sync: true # USE WITH CAUTION. Return to user before pages have been persisted.
|
||||
perm: 0644 # permission to use for the database file and intermediate directories
|
||||
|
||||
tracing:
|
||||
enabled: true
|
||||
exporter: "otlp_grpc"
|
||||
endpoint: "localhost"
|
||||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
|
||||
"github.com/hashicorp/golang-lru/v2/simplelru"
|
||||
"google.golang.org/grpc"
|
||||
|
@ -84,8 +85,15 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn,
|
|||
return nil, err
|
||||
}
|
||||
|
||||
opts := make([]grpc.DialOption, 1, 2)
|
||||
opts[0] = grpc.WithBlock()
|
||||
opts := []grpc.DialOption{
|
||||
grpc.WithBlock(),
|
||||
grpc.WithChainUnaryInterceptor(
|
||||
tracing.NewGRPCUnaryClientInteceptor(),
|
||||
),
|
||||
grpc.WithChainStreamInterceptor(
|
||||
tracing.NewGRPCStreamClientInterceptor(),
|
||||
),
|
||||
}
|
||||
|
||||
// FIXME(@fyrchik): ugly hack #1322
|
||||
if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") {
|
||||
|
|
|
@ -10,6 +10,7 @@ import (
|
|||
"math/rand"
|
||||
"sync"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
|
||||
|
@ -146,7 +147,14 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64,
|
|||
return false
|
||||
}
|
||||
|
||||
cc, err := grpc.DialContext(egCtx, a.URIAddr(), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
cc, err := grpc.DialContext(egCtx, a.URIAddr(),
|
||||
grpc.WithChainUnaryInterceptor(
|
||||
tracing.NewGRPCUnaryClientInteceptor(),
|
||||
),
|
||||
grpc.WithChainStreamInterceptor(
|
||||
tracing.NewGRPCStreamClientInterceptor(),
|
||||
),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
// Failed to connect, try the next address.
|
||||
return false
|
||||
|
|
Loading…
Reference in a new issue