[#369] Write log to file

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>

api/middleware/log_http.go

@@ -3,25 +3,52 @@ package middleware
 import (
 	"io"
 	"net/http"
+	"net/url"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"gopkg.in/natefinch/lumberjack.v2"
 )
 
-type LogHTTPSettings struct {
+const (
+	StdoutPath = "stdout"
+	StderrPath = "stderr"
+	SinkName   = "lumberjack"
+)
+
+type LogHTTPConfig struct {
 	Enabled    bool
 	MaxBody    int64
+	MaxLogSize int
+	OutputPath string
+	sinkPath   string
+	UseGzip    bool
+	fileLogger *zap.Logger
+}
+
+type lumberjackSink struct {
+	*lumberjack.Logger
+}
+
+func (lumberjackSink) Sync() error {
+	return nil
 }
 
 // LogHTTP logs http parameters from s3 request.
-func LogHTTP(l *zap.Logger, settings LogHTTPSettings) Func {
+func LogHTTP(l *zap.Logger, config LogHTTPConfig) Func {
 	return func(h http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			if !settings.Enabled {
+			if !config.Enabled {
 				h.ServeHTTP(w, r)
 				return
 			}
-			var httplog = l.With(
+			var err error
+			config.fileLogger, err = getFileLogger(config)
+			if err != nil {
+				l.Error("error getting file logger", zap.Error(err))
+			}
+			var httplog = config.fileLogger.With(
 				zap.String("from", r.RemoteAddr),
 				zap.String("URI", r.RequestURI),
 				zap.String("method", r.Method),
@@ -29,19 +56,69 @@ func LogHTTP(l *zap.Logger, settings LogHTTPSettings) Func {
 
 			httplog = withFieldIfExist(httplog, "query", r.URL.Query())
 			httplog = withFieldIfExist(httplog, "headers", r.Header)
-			if r.ContentLength != 0 && r.ContentLength <= settings.MaxBody {
+			if r.ContentLength != 0 && r.ContentLength <= config.MaxBody {
 				var err error
 				httplog, err = withBody(httplog, r)
 				if err != nil {
-					l.Error("read body error")
+					l.Error("read body error", zap.Error(err))
 				}
 			}
+
 			httplog.Info(logs.RequestHTTP)
 			h.ServeHTTP(w, r)
 		})
 	}
 }
 
+// getFileLogger returns logger for http requests or create new if not exists.
+func getFileLogger(config LogHTTPConfig) (*zap.Logger, error) {
+	if config.fileLogger != nil {
+		return config.fileLogger, nil
+	}
+	c := zap.NewProductionConfig()
+	c.Level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
+	c.Encoding = "json"
+	c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+	c.ErrorOutputPaths = []string{StderrPath}
+
+	err := configureFileOutput(&config)
+	if err != nil {
+		return nil, err
+	}
+
+	c.OutputPaths = []string{config.sinkPath}
+
+	return c.Build()
+}
+
+// configureFileOutput creates folder structure for file output and registers sink for it
+func configureFileOutput(config *LogHTTPConfig) error {
+	pth := config.OutputPath
+	if pth == StdoutPath || pth == StderrPath || pth == "" {
+		config.sinkPath = pth
+		return nil
+	}
+
+	logRoller := &lumberjack.Logger{
+		Filename: pth,
+		MaxSize:  config.MaxLogSize,
+		Compress: config.UseGzip,
+	}
+	err := zap.RegisterSink(SinkName, func(url *url.URL) (zap.Sink, error) {
+		return lumberjackSink{
+			Logger: logRoller,
+		}, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	config.sinkPath = SinkName + ":" + pth
+
+	return nil
+}
+
+// withBody reads body and attach it to log output.
 func withBody(httplog *zap.Logger, r *http.Request) (*zap.Logger, error) {
 	var body = make([]byte, r.ContentLength)
 	_, err := r.Body.Read(body)
@@ -53,6 +130,7 @@ func withBody(httplog *zap.Logger, r *http.Request) (*zap.Logger, error) {
 	return httplog, nil
 }
 
+// withFieldIfExist checks whether data is not empty and attach it to log output.
 func withFieldIfExist(log *zap.Logger, label string, data map[string][]string) *zap.Logger {
 	if len(data) != 0 {
 		log = log.With(zap.Any(label, data))

api/router.go

@@ -109,7 +109,7 @@ type Config struct {
 	Handler  Handler
 	Center   s3middleware.Center
 	Log      *zap.Logger
-	LogHTTP  s3middleware.LogHTTPSettings
+	LogHTTP  s3middleware.LogHTTPConfig
 	Metrics  *metrics.AppMetrics
 
 	MiddlewareSettings Settings

cmd/s3-gw/app.go

@@ -83,6 +83,7 @@ type (
 
 	appSettings struct {
 		logLevel            zap.AtomicLevel
+		httpLogging         s3middleware.LogHTTPConfig
 		httpLoggingEnabled  bool
 		maxHTTPLogBody      int64
 		maxClient           maxClientsConfig
@@ -192,11 +193,10 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
 	settings := &appSettings{
 		logLevel:            log.lvl,
 		maxClient:           newMaxClients(v),
+		httpLogging:         newHttpLogging(v),
 		defaultMaxAge:       fetchDefaultMaxAge(v, log.logger),
 		reconnectInterval:   fetchReconnectInterval(v),
 		frostfsidValidation: v.GetBool(cfgFrostfsIDValidationEnabled),
-		httpLoggingEnabled:  v.GetBool(cfgLoggerRequestEnabled),
-		maxHTTPLogBody:      v.GetInt64(cfgLoggerRequestMaxBody),
 	}
 
 	settings.resolveZoneList = v.GetStringSlice(cfgResolveBucketAllow)
@@ -565,6 +565,16 @@ func newMaxClients(cfg *viper.Viper) maxClientsConfig {
 	return config
 }
 
+func newHttpLogging(cfg *viper.Viper) s3middleware.LogHTTPConfig {
+	return s3middleware.LogHTTPConfig{
+		Enabled:    cfg.GetBool(cfgHttpLoggingEnabled),
+		MaxBody:    cfg.GetInt64(cfgHttpLoggingMaxBody),
+		MaxLogSize: cfg.GetInt(cfgHttpLoggingMaxLogSize),
+		OutputPath: cfg.GetString(cfgHttpLoggingDestination),
+		UseGzip:    cfg.GetBool(cfgHttpLoggingGzip),
+	}
+}
+
 func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) {
 	var prm pool.InitParameters
 	var prmTree treepool.InitParameters
@@ -686,10 +696,7 @@ func (a *App) Serve(ctx context.Context) {
 		Handler: a.api,
 		Center:  a.ctr,
 		Log:     a.log,
-		LogHTTP: s3middleware.LogHTTPSettings{
+		LogHTTP: a.settings.httpLogging,
-			Enabled: a.settings.httpLoggingEnabled,
-			MaxBody: a.settings.maxHTTPLogBody,
-		},
 		Metrics: a.metrics,
 		Domains: domains,
 

cmd/s3-gw/app_settings.go

@@ -75,8 +75,12 @@ const ( // Settings.
 	cfgLoggerLevel       = "logger.level"
 	cfgLoggerDestination = "logger.destination"
 
-	cfgLoggerRequestEnabled = "http_logging.enabled"
+	// HttpLogging
-	cfgLoggerRequestMaxBody = "http_logging.max_body"
+	cfgHttpLoggingEnabled     = "http_logging.enabled"
+	cfgHttpLoggingMaxBody     = "http_logging.max_body"
+	cfgHttpLoggingMaxLogSize  = "http_logging.max_log_size"
+	cfgHttpLoggingDestination = "http_logging.destination"
+	cfgHttpLoggingGzip        = "http_logging.gzip"
 
 	// Wallet.
 	cfgWalletPath       = "wallet.path"
@@ -719,8 +723,11 @@ func newSettings() *viper.Viper {
 	v.SetDefault(cfgLoggerDestination, "stdout")
 
 	// http logger
-	v.SetDefault(cfgLoggerRequestEnabled, false)
+	v.SetDefault(cfgHttpLoggingEnabled, false)
-	v.SetDefault(cfgLoggerRequestMaxBody, 1024)
+	v.SetDefault(cfgHttpLoggingMaxBody, 1024)
+	v.SetDefault(cfgHttpLoggingMaxLogSize, 50)
+	v.SetDefault(cfgHttpLoggingDestination, "stdout")
+	v.SetDefault(cfgHttpLoggingGzip, false)
 
 	// pool:
 	v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold)

config/config.yaml

@@ -52,6 +52,12 @@ http_logging:
   enabled: false
   # max body size to log
   max_body: 1024
+  # max log size in Mb
+  max_log_size: 20
+  # use log compression
+  gzip: true
+  # possible output values: filesystem path, url, "stdout", "stderr"
+  destination: stdout
   
 # RPC endpoint and order of resolving of bucket names
 rpc_endpoint: http://morph-chain.frostfs.devenv:30333

go.mod

@@ -34,6 +34,7 @@ require (
 	golang.org/x/text v0.14.0
 	google.golang.org/grpc v1.59.0
 	google.golang.org/protobuf v1.33.0
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
 require (

go.sum

@@ -696,6 +696,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=