[#369] Write log to file
Some checks failed
/ DCO (pull_request) Successful in 1m35s
/ Vulncheck (pull_request) Successful in 2m3s
/ Builds (1.20) (pull_request) Successful in 3m10s
/ Builds (1.21) (pull_request) Successful in 2m15s
/ Lint (pull_request) Failing after 3m19s
/ Tests (1.20) (pull_request) Successful in 3m4s
/ Tests (1.21) (pull_request) Successful in 2m57s

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
Nikita Zinkevich 2024-07-15 12:21:10 +03:00
parent 8e2a927334
commit 95c2fa7fda
7 changed files with 120 additions and 19 deletions

View file

@ -3,25 +3,52 @@ package middleware
import (
"io"
"net/http"
"net/url"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"gopkg.in/natefinch/lumberjack.v2"
)
type LogHTTPSettings struct {
Enabled bool
MaxBody int64
const (
StdoutPath = "stdout"
StderrPath = "stderr"
SinkName = "lumberjack"
)
type LogHTTPConfig struct {
Enabled bool
MaxBody int64
MaxLogSize int
OutputPath string
sinkPath string
UseGzip bool
fileLogger *zap.Logger
}
type lumberjackSink struct {
*lumberjack.Logger
}
func (lumberjackSink) Sync() error {
return nil
}
// LogHTTP logs http parameters from s3 request.
func LogHTTP(l *zap.Logger, settings LogHTTPSettings) Func {
func LogHTTP(l *zap.Logger, config LogHTTPConfig) Func {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !settings.Enabled {
if !config.Enabled {
h.ServeHTTP(w, r)
return
}
var httplog = l.With(
var err error
config.fileLogger, err = getFileLogger(config)
if err != nil {
l.Error("error getting file logger", zap.Error(err))
}
var httplog = config.fileLogger.With(
zap.String("from", r.RemoteAddr),
zap.String("URI", r.RequestURI),
zap.String("method", r.Method),
@ -29,19 +56,69 @@ func LogHTTP(l *zap.Logger, settings LogHTTPSettings) Func {
httplog = withFieldIfExist(httplog, "query", r.URL.Query())
httplog = withFieldIfExist(httplog, "headers", r.Header)
if r.ContentLength != 0 && r.ContentLength <= settings.MaxBody {
if r.ContentLength != 0 && r.ContentLength <= config.MaxBody {
var err error
httplog, err = withBody(httplog, r)
if err != nil {
l.Error("read body error")
l.Error("read body error", zap.Error(err))
}
}
httplog.Info(logs.RequestHTTP)
h.ServeHTTP(w, r)
})
}
}
// getFileLogger returns logger for http requests or create new if not exists.
func getFileLogger(config LogHTTPConfig) (*zap.Logger, error) {
if config.fileLogger != nil {
return config.fileLogger, nil
}
c := zap.NewProductionConfig()
c.Level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
c.Encoding = "json"
c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
c.ErrorOutputPaths = []string{StderrPath}
err := configureFileOutput(&config)
if err != nil {
return nil, err
}
c.OutputPaths = []string{config.sinkPath}
return c.Build()
}
// configureFileOutput creates folder structure for file output and registers sink for it
func configureFileOutput(config *LogHTTPConfig) error {
pth := config.OutputPath
if pth == StdoutPath || pth == StderrPath || pth == "" {
config.sinkPath = pth
return nil
}
logRoller := &lumberjack.Logger{
Filename: pth,
MaxSize: config.MaxLogSize,
Compress: config.UseGzip,
}
err := zap.RegisterSink(SinkName, func(url *url.URL) (zap.Sink, error) {
return lumberjackSink{
Logger: logRoller,
}, nil
})
if err != nil {
return err
}
config.sinkPath = SinkName + ":" + pth
return nil
}
// withBody reads body and attach it to log output.
func withBody(httplog *zap.Logger, r *http.Request) (*zap.Logger, error) {
var body = make([]byte, r.ContentLength)
_, err := r.Body.Read(body)
@ -53,6 +130,7 @@ func withBody(httplog *zap.Logger, r *http.Request) (*zap.Logger, error) {
return httplog, nil
}
// withFieldIfExist checks whether data is not empty and attach it to log output.
func withFieldIfExist(log *zap.Logger, label string, data map[string][]string) *zap.Logger {
if len(data) != 0 {
log = log.With(zap.Any(label, data))

View file

@ -109,7 +109,7 @@ type Config struct {
Handler Handler
Center s3middleware.Center
Log *zap.Logger
LogHTTP s3middleware.LogHTTPSettings
LogHTTP s3middleware.LogHTTPConfig
Metrics *metrics.AppMetrics
MiddlewareSettings Settings

View file

@ -83,6 +83,7 @@ type (
appSettings struct {
logLevel zap.AtomicLevel
httpLogging s3middleware.LogHTTPConfig
httpLoggingEnabled bool
maxHTTPLogBody int64
maxClient maxClientsConfig
@ -192,11 +193,10 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
settings := &appSettings{
logLevel: log.lvl,
maxClient: newMaxClients(v),
httpLogging: newHttpLogging(v),
defaultMaxAge: fetchDefaultMaxAge(v, log.logger),
reconnectInterval: fetchReconnectInterval(v),
frostfsidValidation: v.GetBool(cfgFrostfsIDValidationEnabled),
httpLoggingEnabled: v.GetBool(cfgLoggerRequestEnabled),
maxHTTPLogBody: v.GetInt64(cfgLoggerRequestMaxBody),
}
settings.resolveZoneList = v.GetStringSlice(cfgResolveBucketAllow)
@ -565,6 +565,16 @@ func newMaxClients(cfg *viper.Viper) maxClientsConfig {
return config
}
func newHttpLogging(cfg *viper.Viper) s3middleware.LogHTTPConfig {
return s3middleware.LogHTTPConfig{
Enabled: cfg.GetBool(cfgHttpLoggingEnabled),
MaxBody: cfg.GetInt64(cfgHttpLoggingMaxBody),
MaxLogSize: cfg.GetInt(cfgHttpLoggingMaxLogSize),
OutputPath: cfg.GetString(cfgHttpLoggingDestination),
UseGzip: cfg.GetBool(cfgHttpLoggingGzip),
}
}
func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) {
var prm pool.InitParameters
var prmTree treepool.InitParameters
@ -686,10 +696,7 @@ func (a *App) Serve(ctx context.Context) {
Handler: a.api,
Center: a.ctr,
Log: a.log,
LogHTTP: s3middleware.LogHTTPSettings{
Enabled: a.settings.httpLoggingEnabled,
MaxBody: a.settings.maxHTTPLogBody,
},
LogHTTP: a.settings.httpLogging,
Metrics: a.metrics,
Domains: domains,

View file

@ -75,8 +75,12 @@ const ( // Settings.
cfgLoggerLevel = "logger.level"
cfgLoggerDestination = "logger.destination"
cfgLoggerRequestEnabled = "http_logging.enabled"
cfgLoggerRequestMaxBody = "http_logging.max_body"
// HttpLogging
cfgHttpLoggingEnabled = "http_logging.enabled"
cfgHttpLoggingMaxBody = "http_logging.max_body"
cfgHttpLoggingMaxLogSize = "http_logging.max_log_size"
cfgHttpLoggingDestination = "http_logging.destination"
cfgHttpLoggingGzip = "http_logging.gzip"
// Wallet.
cfgWalletPath = "wallet.path"
@ -719,8 +723,11 @@ func newSettings() *viper.Viper {
v.SetDefault(cfgLoggerDestination, "stdout")
// http logger
v.SetDefault(cfgLoggerRequestEnabled, false)
v.SetDefault(cfgLoggerRequestMaxBody, 1024)
v.SetDefault(cfgHttpLoggingEnabled, false)
v.SetDefault(cfgHttpLoggingMaxBody, 1024)
v.SetDefault(cfgHttpLoggingMaxLogSize, 50)
v.SetDefault(cfgHttpLoggingDestination, "stdout")
v.SetDefault(cfgHttpLoggingGzip, false)
// pool:
v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold)

View file

@ -52,6 +52,12 @@ http_logging:
enabled: false
# max body size to log
max_body: 1024
# max log size in Mb
max_log_size: 20
# use log compression
gzip: true
# possible output values: filesystem path, url, "stdout", "stderr"
destination: stdout
# RPC endpoint and order of resolving of bucket names
rpc_endpoint: http://morph-chain.frostfs.devenv:30333

1
go.mod
View file

@ -34,6 +34,7 @@ require (
golang.org/x/text v0.14.0
google.golang.org/grpc v1.59.0
google.golang.org/protobuf v1.33.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
)
require (

2
go.sum
View file

@ -696,6 +696,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=