[#369] Enhanced http requests logging

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>

api/middleware/log_http.go

@@ -0,0 +1,60 @@
+package middleware
+
+import (
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
+	"go.uber.org/zap"
+	"io"
+	"net/http"
+)
+
+type HttpLogSettings interface {
+	GetHttpLogEnabled() bool
+	GetMaxLogBody() int64
+}
+
+// LogHttp logs http parameters from s3 request
+func LogHttp(l *zap.Logger, settings HttpLogSettings) Func {
+	return func(h http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if !settings.GetHttpLogEnabled() {
+				h.ServeHTTP(w, r)
+				return
+			}
+			var httplog = l.With(
+				zap.String("from", r.RemoteAddr),
+				zap.String("URI", r.RequestURI),
+				zap.String("method", r.Method),
+			)
+
+			httplog = withFieldIfExist(httplog, "query", r.URL.Query())
+			httplog = withFieldIfExist(httplog, "headers", r.Header)
+			if r.ContentLength != 0 && r.ContentLength <= settings.GetMaxLogBody() {
+				var err error
+				httplog, err = withBody(httplog, r)
+				if err != nil {
+					l.Error("read body error")
+				}
+			}
+			httplog.Info(logs.RequestHTTP)
+			h.ServeHTTP(w, r)
+		})
+	}
+}
+
+func withBody(httplog *zap.Logger, r *http.Request) (*zap.Logger, error) {
+	var body = make([]byte, r.ContentLength)
+	_, err := r.Body.Read(body)
+	if err != nil && err != io.EOF {
+		return nil, err
+	}
+	httplog = httplog.With(zap.String("body", string(body)))
+
+	return httplog, nil
+}
+
+func withFieldIfExist(log *zap.Logger, label string, data map[string][]string) *zap.Logger {
+	if len(data) != 0 {
+		log = log.With(zap.Any(label, data))
+	}
+	return log
+}

api/router.go

@@ -97,6 +97,7 @@ type Settings interface {
 	s3middleware.RequestSettings
 	s3middleware.PolicySettings
 	s3middleware.MetricsSettings
+	s3middleware.HttpLogSettings
 }
 
 type FrostFSID interface {
@@ -129,6 +130,7 @@ type Config struct {
 func NewRouter(cfg Config) *chi.Mux {
 	api := chi.NewRouter()
 	api.Use(
+		s3middleware.LogHttp(cfg.Log, cfg.MiddlewareSettings),
 		s3middleware.Request(cfg.Log, cfg.MiddlewareSettings),
 		middleware.ThrottleWithOpts(cfg.Throttle),
 		middleware.Recoverer,

cmd/s3-gw/app.go

@@ -83,6 +83,8 @@ type (
 
 	appSettings struct {
 		logLevel            zap.AtomicLevel
+		httpLoggingEnabled  bool
+		maxHttpLogBody      int64
 		maxClient           maxClientsConfig
 		defaultMaxAge       int
 		reconnectInterval   time.Duration
@@ -193,6 +195,8 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
 		defaultMaxAge:       fetchDefaultMaxAge(v, log.logger),
 		reconnectInterval:   fetchReconnectInterval(v),
 		frostfsidValidation: v.GetBool(cfgFrostfsIDValidationEnabled),
+		httpLoggingEnabled:  v.GetBool(cfgLoggerRequestEnabled),
+		maxHttpLogBody:      v.GetInt64(cfgLoggerRequestMaxBody),
 	}
 
 	settings.resolveZoneList = v.GetStringSlice(cfgResolveBucketAllow)
@@ -434,6 +438,18 @@ func (s *appSettings) RetryStrategy() handler.RetryStrategy {
 	return s.retryStrategy
 }
 
+func (s *appSettings) GetMaxLogBody() int64 {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.maxHttpLogBody
+}
+
+func (s *appSettings) GetHttpLogEnabled() bool {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.httpLoggingEnabled
+}
+
 func (a *App) initAPI() {
 	a.initLayer()
 	a.initHandler()

cmd/s3-gw/app_settings.go

@@ -75,6 +75,9 @@ const ( // Settings.
 	cfgLoggerLevel       = "logger.level"
 	cfgLoggerDestination = "logger.destination"
 
+	cfgLoggerRequestEnabled = "http_logging.enabled"
+	cfgLoggerRequestMaxBody = "http_logging.max_body"
+
 	// Wallet.
 	cfgWalletPath       = "wallet.path"
 	cfgWalletAddress    = "wallet.address"
@@ -715,6 +718,10 @@ func newSettings() *viper.Viper {
 	v.SetDefault(cfgLoggerLevel, "debug")
 	v.SetDefault(cfgLoggerDestination, "stdout")
 
+	// http logger
+	v.SetDefault(cfgLoggerRequestEnabled, false)
+	v.SetDefault(cfgLoggerRequestMaxBody, 1024)
+
 	// pool:
 	v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold)
 	v.SetDefault(cfgStreamTimeout, defaultStreamTimeout)

config/config.yaml

@@ -47,6 +47,12 @@ logger:
   level: debug
   destination: stdout
 
+# log http request data (URI, headers, query, etc)
+http_logging:
+  enabled: false
+  # max body size to log
+  max_body: 1024
+  
 # RPC endpoint and order of resolving of bucket names
 rpc_endpoint: http://morph-chain.frostfs.devenv:30333
 resolve_order:

internal/logs/logs.go

@@ -100,6 +100,8 @@ const (
 	FailedToPassAuthentication                           = "failed to pass authentication"                                                                     // Error in ../../api/middleware/auth.go
 	FailedToResolveCID                                   = "failed to resolve CID"                                                                             // Debug in ../../api/middleware/metrics.go
 	RequestStart                                         = "request start"                                                                                     // Info in ../../api/middleware/reqinfo.go
+
+	RequestHTTP                                    = "http request"
 	FailedToUnescapeObjectName                     = "failed to unescape object name"                                                         // Warn in ../../api/middleware/reqinfo.go
 	InvalidDefaultMaxAge                           = "invalid defaultMaxAge"                                                                  // Fatal in ../../cmd/s3-gw/app_settings.go
 	CantShutDownService                            = "can't shut down service"                                                                // Panic in ../../cmd/s3-gw/service.go