[#195] Add tags support
Signed-off-by: Aleksey Kravchenko <al.kravchenko@yadro.com>
This commit is contained in:
Aleksey Kravchenko
parent
76bd6ea40f
commit
6a4d3206bd
22 changed files with 572 additions and 295 deletions
|
|
@ -23,10 +23,8 @@ import (
|
|||
grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||
"git.frostfs.info/TrueCloudLab/zapjournald"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/ssgreg/journald"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
|
|
@ -111,6 +109,11 @@ const (
|
|||
cfgLoggerSamplingThereafter = "logger.sampling.thereafter"
|
||||
cfgLoggerSamplingInterval = "logger.sampling.interval"
|
||||
|
||||
cfgLoggerTags = "logger.tags"
|
||||
cfgLoggerTagsPrefixTmpl = cfgLoggerTags + ".%d."
|
||||
cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "name"
|
||||
cfgLoggerTagsLevelTmpl = cfgLoggerTagsPrefixTmpl + "level"
|
||||
|
||||
// Wallet.
|
||||
cfgWalletPassphrase = "wallet.passphrase"
|
||||
cfgWalletPath = "wallet.path"
|
||||
|
|
@ -193,6 +196,8 @@ var ignore = map[string]struct{}{
|
|||
cmdVersion: {},
|
||||
}
|
||||
|
||||
var defaultTags = []string{logs.TagApp, logs.TagDatapath}
|
||||
|
||||
type Logger struct {
|
||||
logger *zap.Logger
|
||||
lvl zap.AtomicLevel
|
||||
|
|
@ -499,112 +504,33 @@ func mergeConfig(v *viper.Viper, fileName string) error {
|
|||
return v.MergeConfig(cfgFile)
|
||||
}
|
||||
|
||||
type LoggerAppSettings interface {
|
||||
DroppedLogsInc()
|
||||
}
|
||||
func fetchLogTagsConfig(v *viper.Viper, defaultLvl zapcore.Level) (map[string]zapcore.Level, error) {
|
||||
res := make(map[string]zapcore.Level)
|
||||
|
||||
func pickLogger(v *viper.Viper, settings LoggerAppSettings) *Logger {
|
||||
lvl, err := getLogLevel(v)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
for i := 0; ; i++ {
|
||||
name := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i))
|
||||
if name == "" {
|
||||
break
|
||||
}
|
||||
|
||||
lvl := defaultLvl
|
||||
level := v.GetString(fmt.Sprintf(cfgLoggerTagsLevelTmpl, i))
|
||||
if level != "" {
|
||||
if err := lvl.Set(level); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse log tags config, unknown level: '%s'", level)
|
||||
}
|
||||
}
|
||||
|
||||
res[name] = lvl
|
||||
}
|
||||
|
||||
dest := v.GetString(cfgLoggerDestination)
|
||||
|
||||
switch dest {
|
||||
case destinationStdout:
|
||||
return newStdoutLogger(v, lvl, settings)
|
||||
case destinationJournald:
|
||||
return newJournaldLogger(v, lvl, settings)
|
||||
default:
|
||||
panic(fmt.Sprintf("wrong destination for logger: %s", dest))
|
||||
}
|
||||
}
|
||||
|
||||
// newStdoutLogger constructs a zap.Logger instance for current application.
|
||||
// Panics on failure.
|
||||
//
|
||||
// Logger is built from zap's production logging configuration with:
|
||||
// - parameterized level (debug by default)
|
||||
// - console encoding
|
||||
// - ISO8601 time encoding
|
||||
//
|
||||
// Logger records a stack trace for all messages at or above fatal level.
|
||||
//
|
||||
// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace.
|
||||
func newStdoutLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger {
|
||||
stdout := zapcore.AddSync(os.Stderr)
|
||||
level := zap.NewAtomicLevelAt(lvl)
|
||||
|
||||
consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level)
|
||||
consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, settings)
|
||||
|
||||
return &Logger{
|
||||
logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))),
|
||||
lvl: level,
|
||||
}
|
||||
}
|
||||
|
||||
func newJournaldLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger {
|
||||
level := zap.NewAtomicLevelAt(lvl)
|
||||
|
||||
encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields)
|
||||
|
||||
core := zapjournald.NewCore(level, encoder, &journald.Journal{}, zapjournald.SyslogFields)
|
||||
coreWithContext := core.With([]zapcore.Field{
|
||||
zapjournald.SyslogFacility(zapjournald.LogDaemon),
|
||||
zapjournald.SyslogIdentifier(),
|
||||
zapjournald.SyslogPid(),
|
||||
})
|
||||
|
||||
coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, settings)
|
||||
|
||||
return &Logger{
|
||||
logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))),
|
||||
lvl: level,
|
||||
}
|
||||
}
|
||||
|
||||
func newLogEncoder() zapcore.Encoder {
|
||||
c := zap.NewProductionEncoderConfig()
|
||||
c.EncodeTime = zapcore.ISO8601TimeEncoder
|
||||
|
||||
return zapcore.NewConsoleEncoder(c)
|
||||
}
|
||||
|
||||
func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, settings LoggerAppSettings) zapcore.Core {
|
||||
if v.GetBool(cfgLoggerSamplingEnabled) {
|
||||
core = zapcore.NewSamplerWithOptions(core,
|
||||
v.GetDuration(cfgLoggerSamplingInterval),
|
||||
v.GetInt(cfgLoggerSamplingInitial),
|
||||
v.GetInt(cfgLoggerSamplingThereafter),
|
||||
zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) {
|
||||
if dec&zapcore.LogDropped > 0 {
|
||||
settings.DroppedLogsInc()
|
||||
}
|
||||
}))
|
||||
if len(res) == 0 && !v.IsSet(cfgLoggerTags) {
|
||||
for _, tag := range defaultTags {
|
||||
res[tag] = defaultLvl
|
||||
}
|
||||
}
|
||||
|
||||
return core
|
||||
}
|
||||
|
||||
func getLogLevel(v *viper.Viper) (zapcore.Level, error) {
|
||||
var lvl zapcore.Level
|
||||
lvlStr := v.GetString(cfgLoggerLevel)
|
||||
err := lvl.UnmarshalText([]byte(lvlStr))
|
||||
if err != nil {
|
||||
return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+
|
||||
"value should be one of %v", lvlStr, err, [...]zapcore.Level{
|
||||
zapcore.DebugLevel,
|
||||
zapcore.InfoLevel,
|
||||
zapcore.WarnLevel,
|
||||
zapcore.ErrorLevel,
|
||||
zapcore.DPanicLevel,
|
||||
zapcore.PanicLevel,
|
||||
zapcore.FatalLevel,
|
||||
})
|
||||
}
|
||||
return lvl, nil
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func fetchReconnectInterval(cfg *viper.Viper) time.Duration {
|
||||
|
|
@ -620,20 +546,19 @@ func fetchIndexPageTemplate(v *viper.Viper, l *zap.Logger) (string, bool) {
|
|||
if !v.GetBool(cfgIndexPageEnabled) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
reader, err := os.Open(v.GetString(cfgIndexPageTemplatePath))
|
||||
if err != nil {
|
||||
l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err))
|
||||
l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
return "", true
|
||||
}
|
||||
|
||||
tmpl, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err))
|
||||
l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
return "", true
|
||||
}
|
||||
|
||||
l.Info(logs.SetCustomIndexPageTemplate)
|
||||
l.Info(logs.SetCustomIndexPageTemplate, logs.TagField(logs.TagApp))
|
||||
return string(tmpl), true
|
||||
}
|
||||
|
||||
|
|
@ -674,7 +599,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo {
|
|||
}
|
||||
|
||||
if _, ok := seen[serverInfo.Address]; ok {
|
||||
log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address))
|
||||
log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address), logs.TagField(logs.TagApp))
|
||||
continue
|
||||
}
|
||||
seen[serverInfo.Address] = struct{}{}
|
||||
|
|
@ -687,7 +612,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo {
|
|||
func (a *app) initPools(ctx context.Context) {
|
||||
key, err := getFrostFSKey(a.config(), a.log)
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err))
|
||||
a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
var prm pool.InitParameters
|
||||
|
|
@ -695,7 +620,8 @@ func (a *app) initPools(ctx context.Context) {
|
|||
|
||||
prm.SetKey(&key.PrivateKey)
|
||||
prmTree.SetKey(key)
|
||||
a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())))
|
||||
a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())),
|
||||
logs.TagField(logs.TagApp))
|
||||
|
||||
for _, peer := range fetchPeers(a.log, a.config()) {
|
||||
prm.AddNode(peer)
|
||||
|
|
@ -750,11 +676,11 @@ func (a *app) initPools(ctx context.Context) {
|
|||
|
||||
p, err := pool.NewPool(prm)
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err))
|
||||
a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
if err = p.Dial(ctx); err != nil {
|
||||
a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err))
|
||||
a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
if a.config().GetBool(cfgFeaturesTreePoolNetmapSupport) {
|
||||
|
|
@ -763,10 +689,10 @@ func (a *app) initPools(ctx context.Context) {
|
|||
|
||||
treePool, err := treepool.NewPool(prmTree)
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err))
|
||||
a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
if err = treePool.Dial(ctx); err != nil {
|
||||
a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err))
|
||||
a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
a.pool = p
|
||||
|
|
@ -797,7 +723,8 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
|
|||
l.Info(logs.AddedStoragePeer,
|
||||
zap.Int("priority", priority),
|
||||
zap.String("address", address),
|
||||
zap.Float64("weight", weight))
|
||||
zap.Float64("weight", weight),
|
||||
logs.TagField(logs.TagApp))
|
||||
}
|
||||
|
||||
return nodes
|
||||
|
|
@ -836,7 +763,8 @@ func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultV
|
|||
l.Error(logs.InvalidLifetimeUsingDefaultValue,
|
||||
zap.String("parameter", cfgEntry),
|
||||
zap.Duration("value in config", lifetime),
|
||||
zap.Duration("default", defaultValue))
|
||||
zap.Duration("default", defaultValue),
|
||||
logs.TagField(logs.TagApp))
|
||||
} else {
|
||||
return lifetime
|
||||
}
|
||||
|
|
@ -852,7 +780,8 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue
|
|||
l.Error(logs.InvalidCacheSizeUsingDefaultValue,
|
||||
zap.String("parameter", cfgEntry),
|
||||
zap.Int("value in config", size),
|
||||
zap.Int("default", defaultValue))
|
||||
zap.Int("default", defaultValue),
|
||||
logs.TagField(logs.TagApp))
|
||||
} else {
|
||||
return size
|
||||
}
|
||||
|
|
@ -864,7 +793,7 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue
|
|||
func getDialerSource(logger *zap.Logger, cfg *viper.Viper) *internalnet.DialerSource {
|
||||
source, err := internalnet.NewDialerSource(fetchMultinetConfig(cfg, logger))
|
||||
if err != nil {
|
||||
logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err))
|
||||
logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err), logs.TagField(logs.TagApp))
|
||||
}
|
||||
return source
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue