[#369] S3-playback http-body decoding

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>

api/middleware/log_http.go

@@ -21,7 +21,6 @@ type LogHTTPConfig struct {
 	MaxLogSize int
 	OutputPath string
 	UseGzip    bool
-	LogBefore  bool
 }
 
 type fileLogger struct {

cmd/s3-gw/app.go

@@ -573,7 +573,6 @@ func (s *appSettings) updateHTTPLoggingSettings(cfg *viper.Viper, log *zap.Logge
 	s.httpLogging.MaxLogSize = cfg.GetInt(cfgHTTPLoggingMaxLogSize)
 	s.httpLogging.OutputPath = cfg.GetString(cfgHTTPLoggingDestination)
 	s.httpLogging.UseGzip = cfg.GetBool(cfgHTTPLoggingGzip)
-	s.httpLogging.LogBefore = cfg.GetBool(cfgHTTPLoggingLogResponse)
 	if err := s3middleware.ReloadFileLogger(s.httpLogging); err != nil {
 		log.Error(logs.FailedToReloadHTTPFileLogger, zap.Error(err))
 	}

cmd/s3-playback/modules/root.go

@@ -10,12 +10,14 @@ import (
 )
 
 const (
-	configPath        = "config"
-	configPathFlag    = "config"
-	httpTimeout       = "http_timeout"
-	httpTimeoutFlag   = "http-timeout"
-	skipVerifyTLS     = "skip_verify_tls"
-	skipVerifyTLSFlag = "skip-verify-tls"
+	defaultMaxResponseLength = 1024
+	configPath               = "config"
+	configPathFlag           = "config"
+	httpTimeout              = "http_timeout"
+	httpTimeoutFlag          = "http-timeout"
+	skipVerifyTLS            = "skip_verify_tls"
+	skipVerifyTLSFlag        = "skip-verify-tls"
+	printResponseLimit       = "print_response_limit"
 )
 
 var (
@@ -36,6 +38,8 @@ var (
 
 			_ = viper.BindPFlag(skipVerifyTLS, cmd.PersistentFlags().Lookup(skipVerifyTLSFlag))
 
+			viper.SetDefault(printResponseLimit, defaultMaxResponseLength)
+
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, _ []string) error {

cmd/s3-playback/modules/run.go

@@ -18,7 +18,8 @@ import (
 
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
-	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/request"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/playback"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/playback/utils"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -55,26 +56,41 @@ func init() {
 	runCmd.Flags().String(endpointFlag, "", "endpoint URL")
 }
 
-func logResponse(cmd *cobra.Command, id int, resp *http.Response, logReq request.LoggedRequest) {
+func logResponse(cmd *cobra.Command, id int, resp *http.Response, logReq playback.LoggedRequest) {
 	cmd.Println(strconv.Itoa(id)+")", logReq.Method, logReq.URI)
 	cmd.Println(resp.Status)
 	if resp.ContentLength != 0 {
-		body, err := io.ReadAll(io.LimitReader(resp.Body, resp.ContentLength))
+		body := &bytes.Buffer{}
+		isXML, checkBuf, err := utils.DetectXML(resp.Body)
+		if err != nil {
+			cmd.Println(id, err.Error())
+			return
+		}
+
+		resultWriter := utils.ChooseWriter(isXML, body)
+		if _, err = resultWriter.Write(checkBuf); err != nil {
+			cmd.Println(id, err)
+			return
+		}
+
+		_, err = io.Copy(resultWriter, io.LimitReader(resp.Body, viper.GetInt64(printResponseLimit)))
 		if err != nil {
 			cmd.Println(id, err)
+			return
 		}
-		cmd.Println(string(body))
+
+		cmd.Println(body.String())
 	}
 	cmd.Println()
 }
 
 func run(cmd *cobra.Command, _ []string) error {
-	ctx := request.SetCredentials(
+	ctx := playback.SetCredentials(
 		cmd.Context(),
 		viper.GetString(awsAccessKey),
 		viper.GetString(awsSecretKey),
 	)
-	ctx = request.WithMultiparts(ctx)
+	ctx = playback.WithMultiparts(ctx)
 
 	file, err := os.Open(viper.GetString(logPathFlag))
 	if err != nil {
@@ -109,7 +125,7 @@ func run(cmd *cobra.Command, _ []string) error {
 		case <-ctx.Done():
 			return fmt.Errorf("interrupted: %w", ctx.Err())
 		default:
-			resp, err := playback(ctx, logReq, client)
+			resp, err := playbackRequest(ctx, logReq, client)
 			if err != nil {
 				cmd.PrintErrln(strconv.Itoa(id)+")", "failed to playback request:", err)
 				id++
@@ -123,8 +139,8 @@ func run(cmd *cobra.Command, _ []string) error {
 	return nil
 }
 
-func getRequestFromLog(reader *bufio.Reader) (request.LoggedRequest, error) {
-	var logReq request.LoggedRequest
+func getRequestFromLog(reader *bufio.Reader) (playback.LoggedRequest, error) {
+	var logReq playback.LoggedRequest
 	req, err := reader.ReadString('\n')
 	if err != nil {
 		return logReq, err
@@ -138,8 +154,8 @@ func getRequestFromLog(reader *bufio.Reader) (request.LoggedRequest, error) {
 	return logReq, nil
 }
 
-// playback creates http.Request from LoggedRequest and sends it to specified endpoint.
-func playback(ctx context.Context, logReq request.LoggedRequest, client *http.Client) (*http.Response, error) {
+// playbackRequest creates http.Request from LoggedRequest and sends it to specified endpoint.
+func playbackRequest(ctx context.Context, logReq playback.LoggedRequest, client *http.Client) (*http.Response, error) {
 	r, err := prepareRequest(ctx, logReq)
 	if err != nil {
 		return nil, fmt.Errorf("failed to prepare request: %w", err)
@@ -155,7 +171,7 @@ func playback(ctx context.Context, logReq request.LoggedRequest, client *http.Cl
 	}
 	defer resp.Body.Close()
 
-	if err = request.HandleResponse(ctx, r, respBody, logReq.Response); err != nil {
+	if err = playback.HandleResponse(ctx, r, respBody, logReq.Response); err != nil {
 		return nil, fmt.Errorf("failed to register multipart upload: %w", err)
 	}
 	resp.Body = io.NopCloser(bytes.NewBuffer(respBody))
@@ -164,29 +180,29 @@ func playback(ctx context.Context, logReq request.LoggedRequest, client *http.Cl
 }
 
 // prepareRequest creates request from logs and modifies its signature and uploadId (if presents).
-func prepareRequest(ctx context.Context, logReq request.LoggedRequest) (*http.Request, error) {
+func prepareRequest(ctx context.Context, logReq playback.LoggedRequest) (*http.Request, error) {
 	r, err := http.NewRequestWithContext(ctx, logReq.Method, viper.GetString(endpointFlag)+logReq.URI,
-		bytes.NewReader(logReq.Body))
+		bytes.NewReader(logReq.Payload))
 	if err != nil {
 		return nil, err
 	}
 	r.Header = logReq.Header
 
-	err = request.SwapUploadID(ctx, r)
+	err = playback.SwapUploadID(ctx, r)
 	if err != nil {
 		return nil, err
 	}
 
 	sha256hash := sha256.New()
-	sha256hash.Write(logReq.Body)
+	sha256hash.Write(logReq.Payload)
 	r.Header.Set(auth.AmzContentSHA256, hex.EncodeToString(sha256hash.Sum(nil)))
 	if r.Header.Get(api.ContentMD5) != "" {
 		sha256hash.Reset()
 		md5hash := md5.New()
-		md5hash.Write(logReq.Body)
+		md5hash.Write(logReq.Payload)
 		r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(md5hash.Sum(nil)))
 	}
-	err = request.Sign(ctx, r)
+	err = playback.Sign(ctx, r)
 	if err != nil {
 		return nil, err
 	}

config/playback/playback.yaml

@@ -1,7 +1,8 @@
 endpoint: http://localhost:8084
-log: ./log/multipart5.log
+log: ./log/request.log
 credentials:
   access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD
   secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30
 http_timeout: 60s
 skip_verify_tls: false
+print_response_limit: 1024

docs/configuration.md

@@ -389,7 +389,6 @@ http_logging:
   max_log_size: 20
   gzip: true
   destination: stdout
-  log_response: true
 ```
 
 | Parameter      | Type   | SIGHUP reload | Default value | Description                                                                                                                                                                                 |
@@ -399,7 +398,6 @@ http_logging:
 | `max_log_size` | int    | yes           | 50            | Log file size threshold (in megabytes) to be moved in backup file. After reaching  threshold, initial filename is appended with timestamp. And new empty file with initial name is created. |
 | `gzip`         | bool   | yes           | false         | Whether to enable Gzip compression to backup log files.                                                                                                                                     |
 | `destination`  | string | yes           | stdout        | Specify path for log output. Accepts log file path, or "stdout" and "stderr" reserved words to print in output streams. File and folders are created if necessary.                          |
-| `log_response` | bool   | yes           | true          | Whether to attach response body to the log.                                                                                                                                                 |
 
 
 ### `cache` section

docs/playback.md

@@ -31,14 +31,15 @@ If corresponding flag is set, it overrides parameter from config.
 
 ### Configuration parameters
 
-#### Global params
-| # | Config parameter name | Flag name       | Type     | Default value | Description                                            |
-|---|-----------------------|-----------------|----------|---------------|--------------------------------------------------------|
-| 1 | -                     | config          | string   | -             | config file path (e.g. `./config/playback.yaml`)       |
-| 2 | http_timeout          | http-timeout    | duration | 60s           | http request timeout                                   |
-| 3 | skip_verify_tls       | skip-verify-tls | bool     | false         | skips tls certificate verification for https endpoints |
-| 4 | credentials.accessKey | -               | string   | -             | AWS access key id                                      |
-| 5 | credentials.secretKey | -               | string   | -             | AWS secret key                                         |
+#### Global parameters
+| # | Config parameter name | Flag name       | Type     | Default value | Description                                                                   |
+|---|-----------------------|-----------------|----------|---------------|-------------------------------------------------------------------------------|
+| 1 | -                     | config          | string   | -             | config file path (e.g. `./config/playback.yaml`)                              |
+| 2 | http_timeout          | http-timeout    | duration | 60s           | http request timeout                                                          |
+| 3 | skip_verify_tls       | skip-verify-tls | bool     | false         | skips tls certificate verification for https endpoints                        |
+| 4 | credentials.accessKey | -               | string   | -             | AWS access key id                                                             |
+| 5 | credentials.secretKey | -               | string   | -             | AWS secret key                                                                |
+| 6 | print_response_limit  | -               | int      | 1024          | max response length to be printed in stdout, the rest of body will be omitted |
 
 #### `run` command parameters
 | # | Config parameter name | Flag name | Type   | Default value | Description                                            |

playback/request.go

@@ -3,14 +3,17 @@ package playback
 import (
 	"context"
 	"errors"
+	"io"
 	"net/http"
 	"net/url"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/playback/utils"
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 )
@@ -22,14 +25,15 @@ var (
 )
 
 type (
+	httpBody      []byte
 	LoggedRequest struct {
 		From     string      `json:"from"`
 		URI      string      `json:"URI"`
 		Method   string      `json:"method"`
+		Payload  httpBody    `json:"payload,omitempty"`
+		Response httpBody    `json:"response,omitempty"`
 		Query    url.Values  `json:"query"`
-		Body     []byte      `json:"body"`
 		Header   http.Header `json:"headers"`
-		Response []byte      `json:"response"`
 	}
 	Credentials struct {
 		AccessKey string
@@ -39,6 +43,21 @@ type (
 	multipartKey struct{}
 )
 
+func (h *httpBody) UnmarshalJSON(data []byte) error {
+	unquoted, err := strconv.Unquote(string(data))
+	if err != nil {
+		return err
+	}
+	isXML, _, err := utils.DetectXML(strings.NewReader(unquoted))
+	if err != nil {
+		return err
+	}
+	reader := utils.ChooseReader(isXML, strings.NewReader(unquoted))
+	*h, err = io.ReadAll(reader)
+
+	return err
+}
+
 func SetCredentials(ctx context.Context, accessKey, secretKey string) context.Context {
 	return context.WithValue(ctx, contextKey{},
 		Credentials{

playback/utils/utils.go

@@ -35,8 +35,8 @@ func ChooseWriter(isXML bool, body *bytes.Buffer) io.Writer {
 	return body
 }
 
-func ChooseReader(isXml bool, bodyReader io.Reader) io.Reader {
-	if !isXml {
+func ChooseReader(isXML bool, bodyReader io.Reader) io.Reader {
+	if !isXML {
 		return base64.NewDecoder(base64.StdEncoding, bodyReader)
 	}
 	return bodyReader