From b389899447653844288b14d7823b3a11c0e4a480 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 2 Aug 2024 16:45:49 +0300 Subject: [PATCH] [#369] S3-playback http-body decoding Signed-off-by: Nikita Zinkevich --- api/middleware/log_http.go | 1 - cmd/s3-gw/app.go | 1 - cmd/s3-playback/modules/root.go | 16 ++++++---- cmd/s3-playback/modules/run.go | 52 +++++++++++++++++++++------------ config/playback/playback.yaml | 3 +- docs/configuration.md | 2 -- docs/playback.md | 17 ++++++----- playback/request.go | 23 +++++++++++++-- playback/utils/utils.go | 4 +-- 9 files changed, 78 insertions(+), 41 deletions(-) diff --git a/api/middleware/log_http.go b/api/middleware/log_http.go index 8d1512a..5621b43 100644 --- a/api/middleware/log_http.go +++ b/api/middleware/log_http.go @@ -21,7 +21,6 @@ type LogHTTPConfig struct { MaxLogSize int OutputPath string UseGzip bool - LogBefore bool } type fileLogger struct { diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index 1ef37b3..3b6d2b1 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -573,7 +573,6 @@ func (s *appSettings) updateHTTPLoggingSettings(cfg *viper.Viper, log *zap.Logge s.httpLogging.MaxLogSize = cfg.GetInt(cfgHTTPLoggingMaxLogSize) s.httpLogging.OutputPath = cfg.GetString(cfgHTTPLoggingDestination) s.httpLogging.UseGzip = cfg.GetBool(cfgHTTPLoggingGzip) - s.httpLogging.LogBefore = cfg.GetBool(cfgHTTPLoggingLogResponse) if err := s3middleware.ReloadFileLogger(s.httpLogging); err != nil { log.Error(logs.FailedToReloadHTTPFileLogger, zap.Error(err)) } diff --git a/cmd/s3-playback/modules/root.go b/cmd/s3-playback/modules/root.go index ab8d6b2..f1d6f05 100644 --- a/cmd/s3-playback/modules/root.go +++ b/cmd/s3-playback/modules/root.go @@ -10,12 +10,14 @@ import ( ) const ( - configPath = "config" - configPathFlag = "config" - httpTimeout = "http_timeout" - httpTimeoutFlag = "http-timeout" - skipVerifyTLS = "skip_verify_tls" - skipVerifyTLSFlag = "skip-verify-tls" + defaultPrintResponseLimit = 1024 + configPath = "config" + configPathFlag = "config" + httpTimeout = "http_timeout" + httpTimeoutFlag = "http-timeout" + skipVerifyTLS = "skip_verify_tls" + skipVerifyTLSFlag = "skip-verify-tls" + printResponseLimit = "print_response_limit" ) var ( @@ -36,6 +38,8 @@ var ( _ = viper.BindPFlag(skipVerifyTLS, cmd.PersistentFlags().Lookup(skipVerifyTLSFlag)) + viper.SetDefault(printResponseLimit, defaultPrintResponseLimit) + return nil }, RunE: func(cmd *cobra.Command, _ []string) error { diff --git a/cmd/s3-playback/modules/run.go b/cmd/s3-playback/modules/run.go index 53e9bed..c58b6c4 100644 --- a/cmd/s3-playback/modules/run.go +++ b/cmd/s3-playback/modules/run.go @@ -18,7 +18,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth" - "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/request" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/playback" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/playback/utils" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -55,26 +56,41 @@ func init() { runCmd.Flags().String(endpointFlag, "", "endpoint URL") } -func logResponse(cmd *cobra.Command, id int, resp *http.Response, logReq request.LoggedRequest) { +func logResponse(cmd *cobra.Command, id int, resp *http.Response, logReq playback.LoggedRequest) { cmd.Println(strconv.Itoa(id)+")", logReq.Method, logReq.URI) cmd.Println(resp.Status) if resp.ContentLength != 0 { - body, err := io.ReadAll(io.LimitReader(resp.Body, resp.ContentLength)) + body := &bytes.Buffer{} + isXML, checkBuf, err := utils.DetectXML(resp.Body) + if err != nil { + cmd.Println(id, err.Error()) + return + } + + resultWriter := utils.ChooseWriter(isXML, body) + if _, err = resultWriter.Write(checkBuf); err != nil { + cmd.Println(id, err) + return + } + + _, err = io.Copy(resultWriter, io.LimitReader(resp.Body, viper.GetInt64(printResponseLimit))) if err != nil { cmd.Println(id, err) + return } - cmd.Println(string(body)) + + cmd.Println(body.String()) } cmd.Println() } func run(cmd *cobra.Command, _ []string) error { - ctx := request.SetCredentials( + ctx := playback.SetCredentials( cmd.Context(), viper.GetString(awsAccessKey), viper.GetString(awsSecretKey), ) - ctx = request.WithMultiparts(ctx) + ctx = playback.WithMultiparts(ctx) file, err := os.Open(viper.GetString(logPathFlag)) if err != nil { @@ -109,7 +125,7 @@ func run(cmd *cobra.Command, _ []string) error { case <-ctx.Done(): return fmt.Errorf("interrupted: %w", ctx.Err()) default: - resp, err := playback(ctx, logReq, client) + resp, err := playbackRequest(ctx, logReq, client) if err != nil { cmd.PrintErrln(strconv.Itoa(id)+")", "failed to playback request:", err) id++ @@ -123,8 +139,8 @@ func run(cmd *cobra.Command, _ []string) error { return nil } -func getRequestFromLog(reader *bufio.Reader) (request.LoggedRequest, error) { - var logReq request.LoggedRequest +func getRequestFromLog(reader *bufio.Reader) (playback.LoggedRequest, error) { + var logReq playback.LoggedRequest req, err := reader.ReadString('\n') if err != nil { return logReq, err @@ -138,8 +154,8 @@ func getRequestFromLog(reader *bufio.Reader) (request.LoggedRequest, error) { return logReq, nil } -// playback creates http.Request from LoggedRequest and sends it to specified endpoint. -func playback(ctx context.Context, logReq request.LoggedRequest, client *http.Client) (*http.Response, error) { +// playbackRequest creates http.Request from LoggedRequest and sends it to specified endpoint. +func playbackRequest(ctx context.Context, logReq playback.LoggedRequest, client *http.Client) (*http.Response, error) { r, err := prepareRequest(ctx, logReq) if err != nil { return nil, fmt.Errorf("failed to prepare request: %w", err) @@ -155,7 +171,7 @@ func playback(ctx context.Context, logReq request.LoggedRequest, client *http.Cl } defer resp.Body.Close() - if err = request.HandleResponse(ctx, r, respBody, logReq.Response); err != nil { + if err = playback.HandleResponse(ctx, r, respBody, logReq.Response); err != nil { return nil, fmt.Errorf("failed to register multipart upload: %w", err) } resp.Body = io.NopCloser(bytes.NewBuffer(respBody)) @@ -164,29 +180,29 @@ func playback(ctx context.Context, logReq request.LoggedRequest, client *http.Cl } // prepareRequest creates request from logs and modifies its signature and uploadId (if presents). -func prepareRequest(ctx context.Context, logReq request.LoggedRequest) (*http.Request, error) { +func prepareRequest(ctx context.Context, logReq playback.LoggedRequest) (*http.Request, error) { r, err := http.NewRequestWithContext(ctx, logReq.Method, viper.GetString(endpointFlag)+logReq.URI, - bytes.NewReader(logReq.Body)) + bytes.NewReader(logReq.Payload)) if err != nil { return nil, err } r.Header = logReq.Header - err = request.SwapUploadID(ctx, r) + err = playback.SwapUploadID(ctx, r) if err != nil { return nil, err } sha256hash := sha256.New() - sha256hash.Write(logReq.Body) + sha256hash.Write(logReq.Payload) r.Header.Set(auth.AmzContentSHA256, hex.EncodeToString(sha256hash.Sum(nil))) if r.Header.Get(api.ContentMD5) != "" { sha256hash.Reset() md5hash := md5.New() - md5hash.Write(logReq.Body) + md5hash.Write(logReq.Payload) r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(md5hash.Sum(nil))) } - err = request.Sign(ctx, r) + err = playback.Sign(ctx, r) if err != nil { return nil, err } diff --git a/config/playback/playback.yaml b/config/playback/playback.yaml index b7d5b63..27ed4a4 100644 --- a/config/playback/playback.yaml +++ b/config/playback/playback.yaml @@ -1,7 +1,8 @@ endpoint: http://localhost:8084 -log: ./log/multipart5.log +log: ./log/request.log credentials: access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30 http_timeout: 60s skip_verify_tls: false +print_response_limit: 1024 diff --git a/docs/configuration.md b/docs/configuration.md index 38574ec..c9fe9dd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -389,7 +389,6 @@ http_logging: max_log_size: 20 gzip: true destination: stdout - log_response: true ``` | Parameter | Type | SIGHUP reload | Default value | Description | @@ -399,7 +398,6 @@ http_logging: | `max_log_size` | int | yes | 50 | Log file size threshold (in megabytes) to be moved in backup file. After reaching threshold, initial filename is appended with timestamp. And new empty file with initial name is created. | | `gzip` | bool | yes | false | Whether to enable Gzip compression to backup log files. | | `destination` | string | yes | stdout | Specify path for log output. Accepts log file path, or "stdout" and "stderr" reserved words to print in output streams. File and folders are created if necessary. | -| `log_response` | bool | yes | true | Whether to attach response body to the log. | ### `cache` section diff --git a/docs/playback.md b/docs/playback.md index 31d2d7d..0f62c7f 100644 --- a/docs/playback.md +++ b/docs/playback.md @@ -31,14 +31,15 @@ If corresponding flag is set, it overrides parameter from config. ### Configuration parameters -#### Global params -| # | Config parameter name | Flag name | Type | Default value | Description | -|---|-----------------------|-----------------|----------|---------------|--------------------------------------------------------| -| 1 | - | config | string | - | config file path (e.g. `./config/playback.yaml`) | -| 2 | http_timeout | http-timeout | duration | 60s | http request timeout | -| 3 | skip_verify_tls | skip-verify-tls | bool | false | skips tls certificate verification for https endpoints | -| 4 | credentials.accessKey | - | string | - | AWS access key id | -| 5 | credentials.secretKey | - | string | - | AWS secret key | +#### Global parameters +| # | Config parameter name | Flag name | Type | Default value | Description | +|---|-----------------------|-----------------|----------|---------------|-------------------------------------------------------------------------------| +| 1 | - | config | string | - | config file path (e.g. `./config/playback.yaml`) | +| 2 | http_timeout | http-timeout | duration | 60s | http request timeout | +| 3 | skip_verify_tls | skip-verify-tls | bool | false | skips tls certificate verification for https endpoints | +| 4 | credentials.accessKey | - | string | - | AWS access key id | +| 5 | credentials.secretKey | - | string | - | AWS secret key | +| 6 | print_response_limit | - | int | 1024 | max response length to be printed in stdout, the rest of body will be omitted | #### `run` command parameters | # | Config parameter name | Flag name | Type | Default value | Description | diff --git a/playback/request.go b/playback/request.go index d989632..9551dab 100644 --- a/playback/request.go +++ b/playback/request.go @@ -3,14 +3,17 @@ package playback import ( "context" "errors" + "io" "net/http" "net/url" "regexp" + "strconv" "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/playback/utils" v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/aws/aws-sdk-go-v2/credentials" ) @@ -22,14 +25,15 @@ var ( ) type ( + httpBody []byte LoggedRequest struct { From string `json:"from"` URI string `json:"URI"` Method string `json:"method"` + Payload httpBody `json:"payload,omitempty"` + Response httpBody `json:"response,omitempty"` Query url.Values `json:"query"` - Body []byte `json:"body"` Header http.Header `json:"headers"` - Response []byte `json:"response"` } Credentials struct { AccessKey string @@ -39,6 +43,21 @@ type ( multipartKey struct{} ) +func (h *httpBody) UnmarshalJSON(data []byte) error { + unquoted, err := strconv.Unquote(string(data)) + if err != nil { + return err + } + isXML, _, err := utils.DetectXML(strings.NewReader(unquoted)) + if err != nil { + return err + } + reader := utils.ChooseReader(isXML, strings.NewReader(unquoted)) + *h, err = io.ReadAll(reader) + + return err +} + func SetCredentials(ctx context.Context, accessKey, secretKey string) context.Context { return context.WithValue(ctx, contextKey{}, Credentials{ diff --git a/playback/utils/utils.go b/playback/utils/utils.go index 09fbfb5..3e0a03d 100644 --- a/playback/utils/utils.go +++ b/playback/utils/utils.go @@ -35,8 +35,8 @@ func ChooseWriter(isXML bool, body *bytes.Buffer) io.Writer { return body } -func ChooseReader(isXml bool, bodyReader io.Reader) io.Reader { - if !isXml { +func ChooseReader(isXML bool, bodyReader io.Reader) io.Reader { + if !isXML { return base64.NewDecoder(base64.StdEncoding, bodyReader) } return bodyReader