From 074fe474dd7d76203ac72f3ed9b4cdf3d80bc016 Mon Sep 17 00:00:00 2001 From: Aleksey Savaitan Date: Tue, 4 Apr 2023 16:47:59 +0300 Subject: [PATCH] [#3] Initial implementation Signed-off-by: Aleksey Savaitan --- .golangci.yml | 56 +++++++++++++++++ README.md | 115 +++++++++++++++++++++++++++++++++ fields.go | 77 +++++++++++++++++++++++ go.mod | 12 ++++ go.sum | 24 +++++++ zapjournald.go | 168 +++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 452 insertions(+) create mode 100644 .golangci.yml create mode 100644 README.md create mode 100644 fields.go create mode 100644 go.sum create mode 100644 zapjournald.go diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..958f486 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,56 @@ +# This file contains all available configuration options +# with their default values. + +# options for analysis running +run: + # timeout for analysis, e.g. 30s, 5m, default is 1m + timeout: 5m + + # include test files or not, default is true + tests: true + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" + format: tab + +# all available settings of specific linters +linters-settings: + exhaustive: + # indicates that switch statements are to be considered exhaustive if a + # 'default' case is present, even if all enum members aren't listed in the + # switch + default-signifies-exhaustive: true + govet: + # report about shadowed variables + check-shadowing: false + +linters: + enable: + # mandatory linters + - govet + - revive + + # some default golangci-lint linters + - errcheck + - gosimple + - ineffassign + - staticcheck + - typecheck + - unused + + # extra linters + - exhaustive + - godot + - gofmt + - whitespace + - goimports + disable-all: true + fast: false + +issues: + include: + - EXC0002 # should have a comment + - EXC0003 # test/Test ... consider calling this + - EXC0004 # govet + - EXC0005 # C-style breaks \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..69dfec3 --- /dev/null +++ b/README.md @@ -0,0 +1,115 @@ +# Zap Core for systemd Journal +Package `zapjournld` provides `zap` Core for systemd Journal. It supports structured logging. + +Applications may use zap logger to write logs directly into journald and may relatively freely define additional fields, which will be indexed by journald. + +Information about common and custom journald fields is available on https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html. +## Example +```go +package main + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/zapjournald" + "go.uber.org/zap" + + "github.com/ssgreg/journald" + "go.uber.org/zap/zapcore" +) + +func main() { + // StandardLogger + standardLogger, _ := NewStandardLogger("info") + + standardLogger.Info("Simple log raw 1") +} + +func NewStandardLogger(lvlStr string) (*zap.Logger, zap.AtomicLevel) { + lvl, err := getLogLevel(lvlStr) + if err != nil { + panic(err) + } + + zc := zap.NewProductionConfig() + zc.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + zc.Level = zap.NewAtomicLevelAt(lvl) + + // Initialize Zap. + encoder := zapcore.NewJSONEncoder(zc.EncoderConfig) + + core := zapjournald.NewCore(zap.NewAtomicLevelAt(lvl), encoder, &journald.Journal{}, zapjournald.SyslogFields) + coreWithContext := core.With([]zapcore.Field{ + zapjournald.SyslogFacility(zapjournald.LogDaemon), + zapjournald.SyslogIdentifier(), + zapjournald.SyslogPid(), + }) + l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) + return l, zc.Level +} + +func getLogLevel(lvlStr string) (zapcore.Level, error) { + var lvl zapcore.Level + err := lvl.UnmarshalText([]byte(lvlStr)) + if err != nil { + return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+ + "value should be one of %v", lvlStr, err, [...]zapcore.Level{ + zapcore.DebugLevel, + zapcore.InfoLevel, + zapcore.WarnLevel, + zapcore.ErrorLevel, + zapcore.DPanicLevel, + zapcore.PanicLevel, + zapcore.FatalLevel, + }) + } + return lvl, nil +} +``` + +### Results + +#### You can read simple view like this: +``` +sudo journalctl -f +Apr 07 13:06:34 user ___11go_build_git_frostfs_info_TrueCloudLab_zapjournald_main[330983]: {"level":"info","ts":"2023-04-07T13:06:34.990+0300","msg":"Simple log raw 1","SYSLOG_FACILITY":3,"SYSLOG_IDENTIFIER":"___11go_build_git_frostfs_info_TrueCloudLab_zapjournald_main","SYSLOG_PID":330983} +``` +#### Or you can find lines by indexed field +``` +sudo journalctl SYSLOG_PID=76385 +Apr 07 13:06:34 user ___11go_build_git_frostfs_info_TrueCloudLab_zapjournald_main[330983]: {"level":"info","ts":"2023-04-07T13:06:34.990+0300","msg":"Simple log raw 1","SYSLOG_FACILITY":3,"SYSLOG_ID> +``` +#### Or you can read full-structured view like this: +``` +sudo journalctl SYSLOG_PID=76385 --output=json-pretty +{ + "PRIORITY" : "6", + "_SYSTEMD_OWNER_UID" : "1000", + "_HOSTNAME" : "user", + "__MONOTONIC_TIMESTAMP" : "153798818198", + "_SYSTEMD_SLICE" : "user-1000.slice", + "_GID" : "1000", + "_AUDIT_LOGINUID" : "1000", + "_UID" : "1000", + "SYSLOG_IDENTIFIER" : "___11go_build_git_frostfs_info_TrueCloudLab_zapjournald_main", + "__REALTIME_TIMESTAMP" : "1680861994990646", + "_CAP_EFFECTIVE" : "0", + "_COMM" : "___11go_build_g", + "_AUDIT_SESSION" : "59", + "__CURSOR" : "s=9e2d157a286a437ea3405618239c1e07;i=14a96;b=bc1bc632f462476abc9e3e3b0c517f6c;m=23cf1fb996;t=5f8bc2e20bc36;x=9c6c4a86b6da43c", + "_SYSTEMD_USER_SLICE" : "app.slice", + "_SYSTEMD_CGROUP" : "/user.slice/user-1000.slice/user@1000.service/app.slice/app-gnome-jetbrains\\x2dgoland-203587.scope", + "MESSAGE" : "{\"level\":\"info\",\"ts\":\"2023-04-07T13:06:34.990+0300\",\"msg\":\"Simple log raw 1\",\"SYSLOG_FACILITY\":3,\"SYSLOG_IDENTIFIER\":\"___11go_build_git_frostfs_info_TrueCloudLab_zapjournal> + "_SOURCE_REALTIME_TIMESTAMP" : "1680861994990539", + "_MACHINE_ID" : "82be99c92deb4b36850366d7742de698", + "SYSLOG_FACILITY" : "3", + "_BOOT_ID" : "bc1bc632f462476abc9e3e3b0c517f6c", + "_PID" : "330983", + "_SELINUX_CONTEXT" : "unconfined\n", + "_SYSTEMD_UNIT" : "user@1000.service", + "_TRANSPORT" : "journal", + "_SYSTEMD_INVOCATION_ID" : "ed23dc57a96340029ef8bf9956182c20", + "_SYSTEMD_USER_UNIT" : "app-gnome-jetbrains\\x2dgoland-203587.scope", + "SYSLOG_PID" : "330983" +} +``` diff --git a/fields.go b/fields.go new file mode 100644 index 0000000..2917ba7 --- /dev/null +++ b/fields.go @@ -0,0 +1,77 @@ +package zapjournald + +import ( + "os" + "path/filepath" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// Facility defines syslog facility from /usr/include/sys/syslog.h . +type Facility uint32 + +// Syslog compatibility fields. +const ( + SyslogFacilityField = "SYSLOG_FACILITY" + SyslogIdentifierField = "SYSLOG_IDENTIFIER" + SyslogPidField = "SYSLOG_PID" + SyslogTimestampField = "SYSLOG_TIMESTAMP" +) + +// Facilities before LogFtp are the same on Linux, BSD, and OS X. +const ( + LogKern Facility = iota << 3 + LogUser + LogMail + LogDaemon + LogAuth + LogSyslog + LogLpr + LogNews + LogUucp + LogCron + LogAuthpriv + LogFtp + _ // unused + LogAudit // unused + _ // unused + _ // unused + LogLocal0 + LogLocal1 + LogLocal2 + LogLocal3 + LogLocal4 + LogLocal5 + LogLocal6 + LogLocal7 +) + +// LogFacmask is used to extract facility part of the message. +const LogFacmask = 0x03f8 + +// SyslogFields contains slice of fields that +// indexed with syslog by default. +var SyslogFields = []string{ + SyslogFacilityField, + SyslogIdentifierField, + SyslogPidField, + SyslogTimestampField, +} + +func SyslogFacility(facility Facility) zapcore.Field { + return zap.Uint32(SyslogFacilityField, uint32(facility&LogFacmask)>>3) +} + +func SyslogIdentifier() zapcore.Field { + return zap.String(SyslogIdentifierField, filepath.Base(os.Args[0])) +} + +func SyslogPid() zapcore.Field { + return zap.Int(SyslogPidField, os.Getpid()) +} + +func SyslogTimestamp(time time.Time) zapcore.Field { + return zap.Time(SyslogTimestampField, time) +} diff --git a/go.mod b/go.mod index a70b7eb..97b5c37 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,15 @@ module git.frostfs.info/TrueCloudLab/zapjournald go 1.19 + +require ( + github.com/ssgreg/journald v1.0.0 + go.uber.org/zap v1.24.0 + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 +) + +require ( + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/sys v0.1.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d2b09a2 --- /dev/null +++ b/go.sum @@ -0,0 +1,24 @@ +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/ssgreg/journald v1.0.0 h1:0YmTDPJXxcWDPba12qNMdO6TxvfkFSYpFIJ31CwmLcU= +github.com/ssgreg/journald v1.0.0/go.mod h1:RUckwmTM8ghGWPslq2+ZBZzbb9/2KgjzYZ4JEP+oRt0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/zapjournald.go b/zapjournald.go new file mode 100644 index 0000000..168b220 --- /dev/null +++ b/zapjournald.go @@ -0,0 +1,168 @@ +package zapjournald + +import ( + "fmt" + + "github.com/ssgreg/journald" + "go.uber.org/zap/zapcore" + "golang.org/x/exp/maps" +) + +// Core for zapjournald. +// +// Implements zapcore.LevelEnabler and zapcore.Core interfaces. +type Core struct { + zapcore.LevelEnabler + encoder zapcore.Encoder + j *journald.Journal + // field names, which will be stored in journald structure + storedFieldNames map[string]struct{} + // journald fields, which are always present in current core context + contextStructuredFields map[string]interface{} +} + +func NewCore(enab zapcore.LevelEnabler, encoder zapcore.Encoder, journal *journald.Journal, journalFields []string) *Core { + journalFieldsMap := make(map[string]struct{}) + for _, field := range journalFields { + journalFieldsMap[field] = struct{}{} + } + return &Core{ + LevelEnabler: enab, + encoder: encoder, + j: journal, + storedFieldNames: journalFieldsMap, + contextStructuredFields: make(map[string]interface{}), + } +} + +// With adds structured context to the Core. +func (core *Core) With(fields []zapcore.Field) zapcore.Core { + clone := core.clone() + for _, field := range fields { + field.AddTo(clone.encoder) + clone.contextStructuredFields[field.Key] = getFieldValue(field) + } + + return clone +} + +// Check determines whether the supplied Entry should be logged (using the +// embedded LevelEnabler and possibly some extra logic). If the entry +// should be logged, the Core adds itself to the CheckedEntry and returns +// the result. +// +// Callers must use Check before calling Write. +func (core *Core) Check(entry zapcore.Entry, checked *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if core.Enabled(entry.Level) { + return checked.AddCore(entry, core) + } + return checked +} + +// Write serializes the Entry and any Fields supplied at the log site and +// writes them to their destination. +// +// If called, Write should always log the Entry and Fields; it should not +// replicate the logic of Check. +func (core *Core) Write(entry zapcore.Entry, fields []zapcore.Field) error { + // Generate the message. + buffer, err := core.encoder.EncodeEntry(entry, fields) + if err != nil { + return fmt.Errorf("failed to encode log entry: %w", err) + } + + message := buffer.String() + + structuredFields := maps.Clone(core.contextStructuredFields) + for _, field := range fields { + if _, isJournalField := core.storedFieldNames[field.Key]; isJournalField { + structuredFields[field.Key] = getFieldValue(field) + } + } + + // Write the message. + switch entry.Level { + case zapcore.DebugLevel: + return core.j.Send(message, journald.PriorityDebug, structuredFields) + + case zapcore.InfoLevel: + return core.j.Send(message, journald.PriorityInfo, structuredFields) + + case zapcore.WarnLevel: + return core.j.Send(message, journald.PriorityWarning, structuredFields) + + case zapcore.ErrorLevel: + return core.j.Send(message, journald.PriorityErr, structuredFields) + + case zapcore.DPanicLevel: + return core.j.Send(message, journald.PriorityCrit, structuredFields) + + case zapcore.PanicLevel: + return core.j.Send(message, journald.PriorityCrit, structuredFields) + + case zapcore.FatalLevel: + return core.j.Send(message, journald.PriorityCrit, structuredFields) + + default: + return fmt.Errorf("unknown log level: %v", entry.Level) + } +} + +// Sync flushes buffered logs (not used). +func (core *Core) Sync() error { + return nil +} + +// clone returns clone of core. +func (core *Core) clone() *Core { + return &Core{ + LevelEnabler: core.LevelEnabler, + encoder: core.encoder.Clone(), + j: core.j, + storedFieldNames: maps.Clone(core.storedFieldNames), + contextStructuredFields: maps.Clone(core.contextStructuredFields), + } +} + +// getFieldValue returns underlying value stored in zapcore.Field. +func getFieldValue(f zapcore.Field) interface{} { + switch f.Type { + case zapcore.ArrayMarshalerType, + zapcore.ObjectMarshalerType, + zapcore.InlineMarshalerType, + zapcore.BinaryType, + zapcore.BoolType, + zapcore.ByteStringType, + zapcore.Complex128Type, + zapcore.Complex64Type, + zapcore.TimeFullType, + zapcore.ReflectType, + zapcore.NamespaceType, + zapcore.StringerType, + zapcore.ErrorType, + zapcore.SkipType: + return f.Interface + case zapcore.DurationType, + zapcore.Float64Type, + zapcore.Float32Type, + zapcore.Int64Type, + zapcore.Int32Type, + zapcore.Int16Type, + zapcore.Int8Type, + zapcore.Uint64Type, + zapcore.Uint32Type, + zapcore.Uint16Type, + zapcore.Uint8Type, + zapcore.UintptrType: + return f.Integer + case zapcore.StringType: + return f.String + case zapcore.TimeType: + if f.Interface != nil { + return f.Interface + } + return f.Integer + default: + panic(fmt.Sprintf("unknown field type: %v", f)) + } +}