[#1619] logger: Filter entries by tags provided in config

Change-Id: Ia2a79d6cb2a5eb263fb2e6db3f9cf9f2a7d57118
Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>

cmd/frostfs-ir/config.go

@@ -4,12 +4,14 @@ import (
 	"context"
 	"os"
 	"os/signal"
+	"strconv"
 	"syscall"
 
 	configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
 	control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
+	"github.com/spf13/cast"
 	"github.com/spf13/viper"
 	"go.uber.org/zap"
 )
@@ -44,11 +46,30 @@ func reloadConfig() error {
 	if err != nil {
 		return err
 	}
+	err = logPrm.SetTags(loggerTags())
+	if err != nil {
+		return err
+	}
 	log.Reload(logPrm)
 
 	return nil
 }
 
+func loggerTags() [][]string {
+	var res [][]string
+	for i := 0; ; i++ {
+		var item []string
+		index := strconv.FormatInt(int64(i), 10)
+		names := cast.ToString(cfg.Get("logger.tags." + index + ".names"))
+		if names == "" {
+			break
+		}
+		item = append(item, names, cast.ToString(cfg.Get("logger.tags."+index+".level")))
+		res = append(res, item)
+	}
+	return res
+}
+
 func watchForSignal(ctx context.Context, cancel func()) {
 	ch := make(chan os.Signal, 1)
 	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)

cmd/frostfs-ir/main.go

@@ -80,6 +80,8 @@ func main() {
 	exitErr(err)
 	logPrm.SamplingHook = metrics.LogMetrics().GetSamplingHook()
 	logPrm.PrependTimestamp = cfg.GetBool("logger.timestamp")
+	err = logPrm.SetTags(loggerTags())
+	exitErr(err)
 
 	log, err = logger.NewLogger(logPrm)
 	exitErr(err)

cmd/frostfs-node/config.go

@@ -108,6 +108,7 @@ type applicationConfiguration struct {
 		level       string
 		destination string
 		timestamp   bool
+		tags        [][]string
 	}
 
 	ObjectCfg struct {
@@ -232,6 +233,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error {
 	a.LoggerCfg.level = loggerconfig.Level(c)
 	a.LoggerCfg.destination = loggerconfig.Destination(c)
 	a.LoggerCfg.timestamp = loggerconfig.Timestamp(c)
+	a.LoggerCfg.tags = loggerconfig.Tags(c)
 
 	// Object
 
@@ -1090,6 +1092,11 @@ func (c *cfg) loggerPrm() (logger.Prm, error) {
 		return logger.Prm{}, errors.New("incorrect log destination format: " + c.LoggerCfg.destination)
 	}
 	prm.PrependTimestamp = c.LoggerCfg.timestamp
+	err = prm.SetTags(c.LoggerCfg.tags)
+	if err != nil {
+		// not expected since validation should be performed before
+		return logger.Prm{}, errors.New("incorrect allowed tags format: " + c.LoggerCfg.destination)
+	}
 
 	return prm, nil
 }

cmd/frostfs-node/config/logger/config.go

@@ -2,6 +2,7 @@ package loggerconfig
 
 import (
 	"os"
+	"strconv"
 	"time"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
@@ -60,6 +61,23 @@ func Timestamp(c *config.Config) bool {
 	return config.BoolSafe(c.Sub(subsection), "timestamp")
 }
 
+// Tags returns the value of "tags" config parameter from "logger" section.
+func Tags(c *config.Config) [][]string {
+	var res [][]string
+	sub := c.Sub(subsection).Sub("tags")
+	for i := 0; ; i++ {
+		var item []string
+		s := sub.Sub(strconv.FormatInt(int64(i), 10))
+		names := config.StringSafe(s, "names")
+		if names == "" {
+			break
+		}
+		item = append(item, names, config.StringSafe(s, "level"))
+		res = append(res, item)
+	}
+	return res
+}
+
 // ToLokiConfig extracts loki config.
 func ToLokiConfig(c *config.Config) loki.Config {
 	hostname, _ := os.Hostname()

cmd/frostfs-node/validate.go

@@ -30,6 +30,11 @@ func validateConfig(c *config.Config) error {
 		return fmt.Errorf("invalid logger destination: %w", err)
 	}
 
+	err = loggerPrm.SetTags(loggerconfig.Tags(c))
+	if err != nil {
+		return fmt.Errorf("invalid list of allowed tags: %w", err)
+	}
+
 	// shard configuration validation
 
 	shardNum := 0

pkg/util/logger/log.go

@@ -6,21 +6,34 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing"
 	qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 )
 
 func (l *Logger) Debug(ctx context.Context, msg string, fields ...zap.Field) {
+	if l.denyLogEntry(zapcore.DebugLevel) {
+		return
+	}
 	l.z.Debug(msg, appendContext(ctx, fields...)...)
 }
 
 func (l *Logger) Info(ctx context.Context, msg string, fields ...zap.Field) {
+	if l.denyLogEntry(zapcore.InfoLevel) {
+		return
+	}
 	l.z.Info(msg, appendContext(ctx, fields...)...)
 }
 
 func (l *Logger) Warn(ctx context.Context, msg string, fields ...zap.Field) {
+	if l.denyLogEntry(zapcore.WarnLevel) {
+		return
+	}
 	l.z.Warn(msg, appendContext(ctx, fields...)...)
 }
 
 func (l *Logger) Error(ctx context.Context, msg string, fields ...zap.Field) {
+	if l.denyLogEntry(zapcore.ErrorLevel) {
+		return
+	}
 	l.z.Error(msg, appendContext(ctx, fields...)...)
 }
 
@@ -33,3 +46,11 @@ func appendContext(ctx context.Context, fields ...zap.Field) []zap.Field {
 	}
 	return fields
 }
+
+func (l *Logger) denyLogEntry(level zapcore.Level) bool {
+	tl := l.tl.Load().(map[Tag]zapcore.Level)
+	if lvl, ok := tl[l.t]; ok {
+		return level < lvl
+	}
+	return level < l.lvl.Level()
+}

pkg/util/logger/logger.go

@@ -2,6 +2,7 @@ package logger
 
 import (
 	"fmt"
+	"sync/atomic"
 	"time"
 
 	"git.frostfs.info/TrueCloudLab/zapjournald"
@@ -15,6 +16,12 @@ import (
 type Logger struct {
 	z   *zap.Logger
 	lvl zap.AtomicLevel
+	// Tag used by Logger
+	t Tag
+	// Contains map of Tag to log level, overrides lvl
+	tl *atomic.Value
+	// Parent zap.Logger, required to override field zapTagFieldName in the output
+	pz *zap.Logger
 }
 
 // Prm groups Logger's parameters.
@@ -36,12 +43,17 @@ type Prm struct {
 
 	// PrependTimestamp specifies whether to prepend a timestamp in the log
 	PrependTimestamp bool
+
+	// map of tag's bit masks to log level, overrides lvl
+	tl map[Tag]zapcore.Level
 }
 
 const (
 	DestinationUndefined = ""
 	DestinationStdout    = "stdout"
 	DestinationJournald  = "journald"
+
+	zapTagFieldName = "tag"
 )
 
 // SetLevelString sets the minimum logging level. Default is
@@ -65,6 +77,12 @@ func (p *Prm) SetDestination(d string) error {
 	return nil
 }
 
+// SetTags parses list of tags with log level.
+func (p *Prm) SetTags(tags [][]string) (err error) {
+	p.tl, err = parseTags(tags)
+	return err
+}
+
 // NewLogger constructs a new zap logger instance. Constructing with nil
 // parameters is safe: default values will be used then.
 // Passing non-nil parameters after a successful creation (non-error) allows
@@ -88,10 +106,8 @@ func NewLogger(prm Prm) (*Logger, error) {
 }
 
 func newConsoleLogger(prm Prm) (*Logger, error) {
-	lvl := zap.NewAtomicLevelAt(prm.level)
-
 	c := zap.NewProductionConfig()
-	c.Level = lvl
+	c.Level = zap.NewAtomicLevelAt(zapcore.DebugLevel)
 	c.Encoding = "console"
 	if prm.SamplingHook != nil {
 		c.Sampling.Hook = prm.SamplingHook
@@ -110,15 +126,19 @@ func newConsoleLogger(prm Prm) (*Logger, error) {
 	if err != nil {
 		return nil, err
 	}
+	parentZap := *lZap
+	lZap = lZap.With(zap.String(zapTagFieldName, tagToString(TagMain)))
 
-	l := &Logger{z: lZap, lvl: lvl}
+	v := atomic.Value{}
+	v.Store(prm.tl)
+
+	l := &Logger{z: lZap, pz: &parentZap, lvl: zap.NewAtomicLevelAt(prm.level), t: TagMain, tl: &v}
 
 	return l, nil
 }
 
 func newJournaldLogger(prm Prm) (*Logger, error) {
-	lvl := zap.NewAtomicLevelAt(prm.level)
+	lvl := zap.NewAtomicLevelAt(zapcore.DebugLevel)
-
 	c := zap.NewProductionConfig()
 	if prm.SamplingHook != nil {
 		c.Sampling.Hook = prm.SamplingHook
@@ -151,26 +171,48 @@ func newJournaldLogger(prm Prm) (*Logger, error) {
 		samplerOpts...,
 	)
 	lZap := zap.New(samplingCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), zap.AddCallerSkip(1))
+	parentZap := *lZap
+	lZap = lZap.With(zap.String(zapTagFieldName, tagToString(TagMain)))
 
-	l := &Logger{z: lZap, lvl: lvl}
+	v := atomic.Value{}
+	v.Store(prm.tl)
+
+	l := &Logger{z: lZap, pz: &parentZap, lvl: zap.NewAtomicLevelAt(prm.level), t: TagMain, tl: &v}
 
 	return l, nil
 }
 
 func (l *Logger) Reload(prm Prm) {
 	l.lvl.SetLevel(prm.level)
+	l.tl.Store(prm.tl)
 }
 
 func (l *Logger) WithOptions(options ...zap.Option) {
 	l.z = l.z.WithOptions(options...)
+	l.pz = l.pz.WithOptions(options...)
 }
 
 func (l *Logger) With(fields ...zap.Field) *Logger {
-	return &Logger{z: l.z.With(fields...)}
+	c := *l
+	c.z = l.z.With(fields...)
+	return &c
+}
+
+func (l *Logger) WithTag(tag Tag) *Logger {
+	c := *l
+	c.t = tag
+	c.z = c.pz.With(zap.String(zapTagFieldName, tagToString(tag)))
+	return &c
 }
 
 func NewLoggerWrapper(z *zap.Logger) *Logger {
+	tl := &atomic.Value{}
+	tl.Store(make(map[Tag]zapcore.Level))
+
 	return &Logger{
-		z: z.WithOptions(zap.AddCallerSkip(1)),
+		z:   z.WithOptions(zap.AddCallerSkip(1)),
+		pz:  z.WithOptions(zap.AddCallerSkip(1)),
+		tl:  tl,
+		lvl: zap.NewAtomicLevelAt(zapcore.DebugLevel),
 	}
 }

pkg/util/logger/tags.go

@@ -0,0 +1,69 @@
+package logger
+
+import (
+	"fmt"
+	"math"
+	"strings"
+
+	"go.uber.org/zap/zapcore"
+)
+
+type Tag uint8
+
+const (
+	TagMain Tag = iota
+
+	tagMain = "main"
+)
+
+// tagToMask return bit mask for the tag, encoded in uint32.
+func tagFromString(str string) (Tag, error) {
+	switch str {
+	case tagMain:
+		return TagMain, nil
+	default:
+		return math.MaxUint8, fmt.Errorf("unsupported tag %s", str)
+	}
+}
+
+// tagToString return string representation for the tag.
+func tagToString(tag Tag) string {
+	switch tag {
+	case TagMain:
+		return tagMain
+	default:
+		return ""
+	}
+}
+
+// parseTags returns:
+// - map(always instantiated) of tag to custom log level for that tag;
+// - error if it occurred(map is empty).
+func parseTags(raw [][]string) (map[Tag]zapcore.Level, error) {
+	m := make(map[Tag]zapcore.Level)
+	if len(raw) == 0 {
+		return m, nil
+	}
+	for _, item := range raw {
+		str, level := item[0], item[1]
+		if len(level) == 0 {
+			// It is not necessary to parse tags without level,
+			// because default log level will be used.
+			continue
+		}
+		var l zapcore.Level
+		err := l.UnmarshalText([]byte(level))
+		if err != nil {
+			return nil, err
+		}
+		tmp := strings.Split(str, ",")
+		for _, tagStr := range tmp {
+			tag, err := tagFromString(strings.TrimSpace(tagStr))
+			if err != nil {
+				return nil, err
+			}
+			m[tag] = l
+		}
+	}
+	return m, nil
+}