[#369] Add response logging

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>

api/middleware/log_http.go

@@ -22,6 +22,7 @@ type LogHTTPConfig struct {
 	MaxLogSize  int
 	OutputPath  string
 	UseGzip     bool
+	LogResponse bool
 }
 
 type fileLogger struct {
@@ -97,6 +98,23 @@ func ReloadFileLogger(conf *LogHTTPConfig) error {
 	return nil
 }
 
+// responseReadWriter helps read http response body.
+type responseReadWriter struct {
+	http.ResponseWriter
+	response *bytes.Buffer
+}
+
+func (ww *responseReadWriter) Write(data []byte) (int, error) {
+	ww.response.Write(data)
+	return ww.ResponseWriter.Write(data)
+}
+
+func (ww *responseReadWriter) Flush() {
+	if f, ok := ww.ResponseWriter.(http.Flusher); ok {
+		f.Flush()
+	}
+}
+
 // LogHTTP logs http parameters from s3 request.
 func LogHTTP(l *zap.Logger, config *LogHTTPConfig) Func {
 	var err error
@@ -129,8 +147,18 @@ func LogHTTP(l *zap.Logger, config *LogHTTPConfig) Func {
 					l.Warn(logs.FailedToGetRequestBody, zap.Error(err))
 				}
 			}
-			httplog.Info(logs.RequestHTTP)
+			if !config.LogResponse {
+				httplog.Info(logs.LogHTTP)
 				h.ServeHTTP(w, r)
+			} else {
+				var resp = bytes.Buffer{}
+				ww := &responseReadWriter{ResponseWriter: w, response: &resp}
+				h.ServeHTTP(ww, r)
+				if int64(resp.Len()) <= config.MaxBody<<20 {
+					httplog.With(zap.String("response", base64.StdEncoding.EncodeToString(resp.Bytes()))).
+						Info(logs.LogHTTP)
+				}
+			}
 		})
 	}
 }

api/middleware/log_http_stub.go

@@ -15,6 +15,7 @@ type LogHTTPConfig struct {
 	MaxLogSize  int
 	OutputPath  string
 	UseGzip     bool
+	LogResponse bool
 }
 
 func LogHTTP(l *zap.Logger, _ *LogHTTPConfig) Func {

cmd/s3-gw/app.go

@@ -573,6 +573,7 @@ func (s *appSettings) updateHTTPLoggingSettings(cfg *viper.Viper, log *zap.Logge
 	s.httpLogging.MaxLogSize = cfg.GetInt(cfgHTTPLoggingMaxLogSize)
 	s.httpLogging.OutputPath = cfg.GetString(cfgHTTPLoggingDestination)
 	s.httpLogging.UseGzip = cfg.GetBool(cfgHTTPLoggingGzip)
+	s.httpLogging.LogResponse = cfg.GetBool(cfgHTTPLoggingLogResponse)
 	if err := s3middleware.ReloadFileLogger(s.httpLogging); err != nil {
 		log.Error(logs.FailedToReloadHTTPFileLogger, zap.Error(err))
 	}

cmd/s3-gw/app_settings.go

@@ -81,6 +81,7 @@ const ( // Settings.
 	cfgHTTPLoggingMaxLogSize  = "http_logging.max_log_size"
 	cfgHTTPLoggingDestination = "http_logging.destination"
 	cfgHTTPLoggingGzip        = "http_logging.gzip"
+	cfgHTTPLoggingLogResponse = "http_logging.log_response"
 
 	// Wallet.
 	cfgWalletPath       = "wallet.path"
@@ -728,6 +729,7 @@ func newSettings() *viper.Viper {
 	v.SetDefault(cfgHTTPLoggingMaxLogSize, 50)
 	v.SetDefault(cfgHTTPLoggingDestination, "stdout")
 	v.SetDefault(cfgHTTPLoggingGzip, false)
+	v.SetDefault(cfgHTTPLoggingLogResponse, true)
 
 	// pool:
 	v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold)

docs/configuration.md

@@ -384,6 +384,7 @@ http_logging:
   max_log_size: 20
   gzip: true
   destination: stdout
+  log_response: true
 ```
 
 | Parameter      | Type   | SIGHUP reload | Default value | Description                                                                                                                                                                                 |
@@ -391,8 +392,9 @@ http_logging:
 | `enabled`      | bool   | yes           | false         | Flag to enable the logger.                                                                                                                                                                  |
 | `max_body`     | int    | yes           | 10            | Max size for request and response bodies that will be attached to the log (in megabytes).                                                                                                   |
 | `max_log_size` | int    | yes           | 50            | Log file size threshold (in megabytes) to be moved in backup file. After reaching  threshold, initial filename is appended with timestamp. And new empty file with initial name is created. |
-| `gzip`         | bool   | yes           | false         | Whether to enable Gzip compression for backup log files.                                                                                                                                    |
+| `gzip`         | bool   | yes           | false         | Whether to enable Gzip compression to backup log files.                                                                                                                                     |
-| `destination`  | string | yes           | stdout        | Specify path for log output. Accepts log file path, or "stdout" and "stderr" reserved words to print in output streams.                                                                     |
+| `destination`  | string | yes           | stdout        | Specify path for log output. Accepts log file path, or "stdout" and "stderr" reserved words to print in output streams. File and folders are creating if necessary.                         |
+| `log_response` | bool   | yes           | true          | Whether to attach response body to the log.                                                                                                                                                 |
 
 
 ### `cache` section

internal/logs/logs.go

@@ -100,7 +100,7 @@ const (
 	FailedToPassAuthentication                           = "failed to pass authentication"                                                                     // Error in ../../api/middleware/auth.go
 	FailedToResolveCID                                   = "failed to resolve CID"                                                                             // Debug in ../../api/middleware/metrics.go
 	RequestStart                                         = "request start"                                                                                     // Info in ../../api/middleware/reqinfo.go
-	RequestHTTP                                          = "http request"                                                                                      // Info in ../../api/middleware/log_http.go
+	LogHTTP                                              = "http log"                                                                                          // Info in ../../api/middleware/log_http.go
 	FailedToInitializeHTTPLogger                         = "failed to initialize http logger"                                                                  // Warn in ../../api/middleware/log_http.go
 	FailedToReloadHTTPFileLogger                         = "failed to reload http file logger"                                                                 // Warn in ../../api/middleware/log_http.go
 	FailedToGetRequestBody                               = "failed to get request body"                                                                        // Warn in ../../api/middleware/log_http.go