[#1619] logger: Filter entries by tags provided in config

Change-Id: Ia2a79d6cb2a5eb263fb2e6db3f9cf9f2a7d57118
Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>

cmd/frostfs-ir/config.go

@@ -4,12 +4,14 @@ import (
 	"context"
 	"os"
 	"os/signal"
+	"strconv"
 	"syscall"
 
 	configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
 	control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
+	"github.com/spf13/cast"
 	"github.com/spf13/viper"
 	"go.uber.org/zap"
 )
@@ -44,11 +46,30 @@ func reloadConfig() error {
 	if err != nil {
 		return err
 	}
-	log.Reload(logPrm)
+	err = logPrm.SetTags(loggerTags())
+	if err != nil {
+		return err
+	}
+	logger.UpdateLevelForTags(logPrm)
 
 	return nil
 }
 
+func loggerTags() [][]string {
+	var res [][]string
+	for i := 0; ; i++ {
+		var item []string
+		index := strconv.FormatInt(int64(i), 10)
+		names := cast.ToString(cfg.Get("logger.tags." + index + ".names"))
+		if names == "" {
+			break
+		}
+		item = append(item, names, cast.ToString(cfg.Get("logger.tags."+index+".level")))
+		res = append(res, item)
+	}
+	return res
+}
+
 func watchForSignal(ctx context.Context, cancel func()) {
 	ch := make(chan os.Signal, 1)
 	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)

cmd/frostfs-ir/main.go

@@ -80,10 +80,14 @@ func main() {
 	exitErr(err)
 	logPrm.SamplingHook = metrics.LogMetrics().GetSamplingHook()
 	logPrm.PrependTimestamp = cfg.GetBool("logger.timestamp")
+	err = logPrm.SetTags(loggerTags())
+	exitErr(err)
 
 	log, err = logger.NewLogger(logPrm)
 	exitErr(err)
 
+	logger.UpdateLevelForTags(logPrm)
+
 	ctx, cancel := context.WithCancel(context.Background())
 
 	pprofCmp = newPprofComponent()

cmd/frostfs-node/config.go

@@ -109,6 +109,7 @@ type applicationConfiguration struct {
 		destination string
 		timestamp   bool
 		options     []zap.Option
+		tags        [][]string
 	}
 
 	ObjectCfg struct {
@@ -241,6 +242,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error {
 		})}
 	}
 	a.LoggerCfg.options = opts
+	a.LoggerCfg.tags = loggerconfig.Tags(c)
 
 	// Object
 
@@ -727,6 +729,7 @@ func initCfg(appCfg *config.Config) *cfg {
 	logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook()
 	log, err := logger.NewLogger(logPrm)
 	fatalOnErr(err)
+	logger.UpdateLevelForTags(logPrm)
 
 	c.internals = initInternals(appCfg, log)
 
@@ -1094,6 +1097,11 @@ func (c *cfg) loggerPrm() (logger.Prm, error) {
 	}
 	prm.PrependTimestamp = c.LoggerCfg.timestamp
 	prm.Options = c.LoggerCfg.options
+	err = prm.SetTags(c.LoggerCfg.tags)
+	if err != nil {
+		// not expected since validation should be performed before
+		return logger.Prm{}, errors.New("incorrect allowed tags format: " + c.LoggerCfg.destination)
+	}
 
 	return prm, nil
 }
@@ -1381,7 +1389,7 @@ func (c *cfg) getComponents(ctx context.Context) []dCmp {
 		if err != nil {
 			return err
 		}
-		c.log.Reload(prm)
+		logger.UpdateLevelForTags(prm)
 		return nil
 	}})
 	components = append(components, dCmp{"runtime", func() error {

cmd/frostfs-node/config/logger/config.go

@@ -2,6 +2,7 @@ package loggerconfig
 
 import (
 	"os"
+	"strconv"
 	"time"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
@@ -60,6 +61,21 @@ func Timestamp(c *config.Config) bool {
 	return config.BoolSafe(c.Sub(subsection), "timestamp")
 }
 
+// Tags returns the value of "tags" config parameter from "logger" section.
+func Tags(c *config.Config) [][]string {
+	var res [][]string
+	sub := c.Sub(subsection).Sub("tags")
+	for i := 0; ; i++ {
+		s := sub.Sub(strconv.FormatInt(int64(i), 10))
+		names := config.StringSafe(s, "names")
+		if names == "" {
+			break
+		}
+		res = append(res, []string{names, config.StringSafe(s, "level")})
+	}
+	return res
+}
+
 // ToLokiConfig extracts loki config.
 func ToLokiConfig(c *config.Config) loki.Config {
 	hostname, _ := os.Hostname()

cmd/frostfs-node/validate.go

@@ -30,6 +30,11 @@ func validateConfig(c *config.Config) error {
 		return fmt.Errorf("invalid logger destination: %w", err)
 	}
 
+	err = loggerPrm.SetTags(loggerconfig.Tags(c))
+	if err != nil {
+		return fmt.Errorf("invalid list of allowed tags: %w", err)
+	}
+
 	// shard configuration validation
 
 	shardNum := 0

pkg/util/logger/logger.go

@@ -13,8 +13,10 @@ import (
 // Logger represents a component
 // for writing messages to log.
 type Logger struct {
-	z   *zap.Logger
-	lvl zap.AtomicLevel
+	z *zap.Logger
+	c zapcore.Core
+	t Tag
+	w bool
 }
 
 // Prm groups Logger's parameters.
@@ -39,6 +41,9 @@ type Prm struct {
 
 	// Options for zap.Logger
 	Options []zap.Option
+
+	// map of tag's bit masks to log level, overrides lvl
+	tl map[Tag]zapcore.Level
 }
 
 const (
@@ -68,6 +73,12 @@ func (p *Prm) SetDestination(d string) error {
 	return nil
 }
 
+// SetTags parses list of tags with log level.
+func (p *Prm) SetTags(tags [][]string) (err error) {
+	p.tl, err = parseTags(tags)
+	return err
+}
+
 // NewLogger constructs a new zap logger instance. Constructing with nil
 // parameters is safe: default values will be used then.
 // Passing non-nil parameters after a successful creation (non-error) allows
@@ -91,10 +102,8 @@ func NewLogger(prm Prm) (*Logger, error) {
 }
 
 func newConsoleLogger(prm Prm) (*Logger, error) {
-	lvl := zap.NewAtomicLevelAt(prm.level)
-
 	c := zap.NewProductionConfig()
-	c.Level = lvl
+	c.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
 	c.Encoding = "console"
 	if prm.SamplingHook != nil {
 		c.Sampling.Hook = prm.SamplingHook
@@ -115,15 +124,13 @@ func newConsoleLogger(prm Prm) (*Logger, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	l := &Logger{z: lZap, lvl: lvl}
+	l := &Logger{z: lZap, c: lZap.Core()}
+	l = l.WithTag(TagMain)
 
 	return l, nil
 }
 
 func newJournaldLogger(prm Prm) (*Logger, error) {
-	lvl := zap.NewAtomicLevelAt(prm.level)
-
 	c := zap.NewProductionConfig()
 	if prm.SamplingHook != nil {
 		c.Sampling.Hook = prm.SamplingHook
@@ -137,7 +144,7 @@ func newJournaldLogger(prm Prm) (*Logger, error) {
 
 	encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields)
 
-	core := zapjournald.NewCore(lvl, encoder, &journald.Journal{}, zapjournald.SyslogFields)
+	core := zapjournald.NewCore(zap.NewAtomicLevelAt(zap.DebugLevel), encoder, &journald.Journal{}, zapjournald.SyslogFields)
 	coreWithContext := core.With([]zapcore.Field{
 		zapjournald.SyslogFacility(zapjournald.LogDaemon),
 		zapjournald.SyslogIdentifier(),
@@ -161,22 +168,75 @@ func newJournaldLogger(prm Prm) (*Logger, error) {
 	}
 	opts = append(opts, prm.Options...)
 	lZap := zap.New(samplingCore, opts...)
-
-	l := &Logger{z: lZap, lvl: lvl}
+	l := &Logger{z: lZap, c: lZap.Core()}
+	l = l.WithTag(TagMain)
 
 	return l, nil
 }
 
-func (l *Logger) Reload(prm Prm) {
-	l.lvl.SetLevel(prm.level)
+// With create a child logger with new fields, don't affect the parent.
+// Throws panic if tag is unset.
+func (l *Logger) With(fields ...zap.Field) *Logger {
+	if l.t == 0 {
+		panic("tag is unset")
+	}
+	c := *l
+	c.z = l.z.With(fields...)
+	// With called under the logger
+	c.w = true
+	return &c
 }
 
-func (l *Logger) With(fields ...zap.Field) *Logger {
-	return &Logger{z: l.z.With(fields...)}
+type core struct {
+	c zapcore.Core
+	l zap.AtomicLevel
+}
+
+func (c *core) Enabled(lvl zapcore.Level) bool {
+	return c.l.Enabled(lvl)
+}
+
+func (c *core) With(fields []zapcore.Field) zapcore.Core {
+	clone := *c
+	clone.c = clone.c.With(fields)
+	return &clone
+}
+
+func (c *core) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry {
+	return c.c.Check(e, ce)
+}
+
+func (c *core) Write(e zapcore.Entry, fields []zapcore.Field) error {
+	return c.c.Write(e, fields)
+}
+
+func (c *core) Sync() error {
+	return c.c.Sync()
+}
+
+// WithTag is an equivalent of calling [NewLogger] with the same parameters for the current logger.
+// Throws panic if provided unsupported tag.
+func (l *Logger) WithTag(tag Tag) *Logger {
+	if tag == 0 || tag > Tag(len(_Tag_index)-1) {
+		panic("unsupported tag " + tag.String())
+	}
+	if l.w {
+		panic("unsupported operation for the logger's state")
+	}
+	c := *l
+	c.t = tag
+	c.z = l.z.WithOptions(zap.WrapCore(func(zapcore.Core) zapcore.Core {
+		return &core{
+			c: l.c.With([]zap.Field{zap.String("tag", tag.String())}),
+			l: tagToLogLevel[tag],
+		}
+	}))
+	return &c
 }
 
 func NewLoggerWrapper(z *zap.Logger) *Logger {
 	return &Logger{
 		z: z.WithOptions(zap.AddCallerSkip(1)),
+		t: TagMain,
 	}
 }

pkg/util/logger/tag_string.go

@@ -0,0 +1,24 @@
+// Code generated by "stringer -type Tag -linecomment"; DO NOT EDIT.
+
+package logger
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[TagMain-1]
+}
+
+const _Tag_name = "main"
+
+var _Tag_index = [...]uint8{0, 4}
+
+func (i Tag) String() string {
+	i -= 1
+	if i >= Tag(len(_Tag_index)-1) {
+		return "Tag(" + strconv.FormatInt(int64(i+1), 10) + ")"
+	}
+	return _Tag_name[_Tag_index[i]:_Tag_index[i+1]]
+}

pkg/util/logger/tags.go

@@ -0,0 +1,75 @@
+package logger
+
+import (
+	"fmt"
+	"strings"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+//go:generate stringer -type Tag -linecomment
+
+type Tag uint8
+
+const (
+	_       Tag = iota //
+	TagMain            // main
+
+	defaultLevel = zapcore.InfoLevel
+)
+
+var (
+	tagToLogLevel = map[Tag]zap.AtomicLevel{}
+	stringToTag   = map[string]Tag{}
+)
+
+func init() {
+	for i := TagMain; i <= Tag(len(_Tag_index)-1); i++ {
+		tagToLogLevel[i] = zap.NewAtomicLevelAt(defaultLevel)
+		stringToTag[i.String()] = i
+	}
+}
+
+// parseTags returns:
+// - map(always instantiated) of tag to custom log level for that tag;
+// - error if it occurred(map is empty).
+func parseTags(raw [][]string) (map[Tag]zapcore.Level, error) {
+	m := make(map[Tag]zapcore.Level)
+	if len(raw) == 0 {
+		return m, nil
+	}
+	for _, item := range raw {
+		str, level := item[0], item[1]
+		if len(level) == 0 {
+			// It is not necessary to parse tags without level,
+			// because default log level will be used.
+			continue
+		}
+		var l zapcore.Level
+		err := l.UnmarshalText([]byte(level))
+		if err != nil {
+			return nil, err
+		}
+		tmp := strings.Split(str, ",")
+		for _, tagStr := range tmp {
+			tag, ok := stringToTag[strings.TrimSpace(tagStr)]
+			if !ok {
+				return nil, fmt.Errorf("unsupported tag %s", str)
+			}
+			m[tag] = l
+		}
+	}
+	return m, nil
+}
+
+func UpdateLevelForTags(prm Prm) {
+	for k, v := range tagToLogLevel {
+		nk, ok := prm.tl[k]
+		if ok {
+			v.SetLevel(nk)
+		} else {
+			v.SetLevel(prm.level)
+		}
+	}
+}