From 364f835b7e203fb96805499ec6a50a30efcdbcf8 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 13 Nov 2023 11:41:11 +0300 Subject: [PATCH] [#740] logs: Add Loki Signed-off-by: Alexander Chuprov --- cmd/frostfs-node/config.go | 8 ++++ cmd/frostfs-node/config/logger/config.go | 54 +++++++++++++++++++++++- go.mod | 2 +- go.sum | 4 +- 4 files changed, 63 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index a41b73d92..a43b59ab2 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -62,6 +62,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/state" + "git.frostfs.info/TrueCloudLab/frostfs-observability/logging/lokicore" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -72,6 +73,7 @@ import ( "github.com/panjf2000/ants/v2" "go.etcd.io/bbolt" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "google.golang.org/grpc" ) @@ -577,6 +579,12 @@ func initCfg(appCfg *config.Config) *cfg { logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook() log, err := logger.NewLogger(logPrm) fatalOnErr(err) + if loggerconfig.ToLokiConfig(appCfg).Enabled { + log.Logger = log.Logger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { + lokiCore := lokicore.New(core, loggerconfig.ToLokiConfig(appCfg)) + return lokiCore + })) + } c.internals = initInternals(appCfg, log) diff --git a/cmd/frostfs-node/config/logger/config.go b/cmd/frostfs-node/config/logger/config.go index 78e4377a9..02645c543 100644 --- a/cmd/frostfs-node/config/logger/config.go +++ b/cmd/frostfs-node/config/logger/config.go @@ -1,12 +1,21 @@ package loggerconfig import ( + "os" + "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "git.frostfs.info/TrueCloudLab/frostfs-observability/logging/lokicore/loki" ) const ( // LevelDefault is a default logger level. - LevelDefault = "info" + LevelDefault = "info" + subsection = "logger" + lokiSubsection = "loki" + AddressDefault = "localhost:3100" + BatchEntriesNumberDefault = 100 + BatchWaitDefault = time.Second ) // Level returns the value of "level" config parameter @@ -15,7 +24,7 @@ const ( // Returns LevelDefault if the value is not a non-empty string. func Level(c *config.Config) string { v := config.StringSafe( - c.Sub("logger"), + c.Sub(subsection), "level", ) if v != "" { @@ -24,3 +33,44 @@ func Level(c *config.Config) string { return LevelDefault } + +// ToLokiConfig extracts loki config. +func ToLokiConfig(c *config.Config) loki.Config { + hostname, _ := os.Hostname() + return loki.Config{ + Enabled: config.BoolSafe(c.Sub(subsection).Sub(lokiSubsection), "enabled"), + BatchWait: getBatchWait(c), + BatchEntriesNumber: getBatchEntriesNumber(c), + Endpoint: getEndpoint(c), + Labels: map[string]string{ + "hostname": hostname, + }, + } +} + +func getBatchWait(c *config.Config) time.Duration { + v := config.DurationSafe(c.Sub(subsection).Sub(lokiSubsection), "max_batch_delay") + if v > 0 { + return v + } + + return BatchWaitDefault +} + +func getBatchEntriesNumber(c *config.Config) int { + v := config.IntSafe(c.Sub(subsection).Sub(lokiSubsection), "max_batch_size") + if v > 0 { + return int(v) + } + + return BatchEntriesNumberDefault +} + +func getEndpoint(c *config.Config) string { + v := config.StringSafe(c.Sub(subsection).Sub(lokiSubsection), "endpoint") + if v != "" { + return v + } + + return AddressDefault +} diff --git a/go.mod b/go.mod index 4f6505e87..5c22fdfc2 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989 - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231101082425-5eee1a733432 diff --git a/go.sum b/go.sum index 05a870b80..806648251 100644 --- a/go.sum +++ b/go.sum @@ -730,8 +730,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845a git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989/go.mod h1:PxsAxe6J4hyYJoU6hyYxD9q5K/DFxdquKHL1BrRaGkg= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d h1:KVHsNsEoedQ1+f+eDq88AFKkLePGfnygt4bcUQrCgPw= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=