[#369] S3-playback http-body decoding
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
parent
f6a7beb2b5
commit
b389899447
9 changed files with 78 additions and 41 deletions
|
|
@ -21,7 +21,6 @@ type LogHTTPConfig struct {
|
|||
MaxLogSize int
|
||||
OutputPath string
|
||||
UseGzip bool
|
||||
LogBefore bool
|
||||
}
|
||||
|
||||
type fileLogger struct {
|
||||
|
|
|
|||
|
|
@ -573,7 +573,6 @@ func (s *appSettings) updateHTTPLoggingSettings(cfg *viper.Viper, log *zap.Logge
|
|||
s.httpLogging.MaxLogSize = cfg.GetInt(cfgHTTPLoggingMaxLogSize)
|
||||
s.httpLogging.OutputPath = cfg.GetString(cfgHTTPLoggingDestination)
|
||||
s.httpLogging.UseGzip = cfg.GetBool(cfgHTTPLoggingGzip)
|
||||
s.httpLogging.LogBefore = cfg.GetBool(cfgHTTPLoggingLogResponse)
|
||||
if err := s3middleware.ReloadFileLogger(s.httpLogging); err != nil {
|
||||
log.Error(logs.FailedToReloadHTTPFileLogger, zap.Error(err))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,12 +10,14 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
configPath = "config"
|
||||
configPathFlag = "config"
|
||||
httpTimeout = "http_timeout"
|
||||
httpTimeoutFlag = "http-timeout"
|
||||
skipVerifyTLS = "skip_verify_tls"
|
||||
skipVerifyTLSFlag = "skip-verify-tls"
|
||||
defaultPrintResponseLimit = 1024
|
||||
configPath = "config"
|
||||
configPathFlag = "config"
|
||||
httpTimeout = "http_timeout"
|
||||
httpTimeoutFlag = "http-timeout"
|
||||
skipVerifyTLS = "skip_verify_tls"
|
||||
skipVerifyTLSFlag = "skip-verify-tls"
|
||||
printResponseLimit = "print_response_limit"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -36,6 +38,8 @@ var (
|
|||
|
||||
_ = viper.BindPFlag(skipVerifyTLS, cmd.PersistentFlags().Lookup(skipVerifyTLSFlag))
|
||||
|
||||
viper.SetDefault(printResponseLimit, defaultPrintResponseLimit)
|
||||
|
||||
return nil
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@ import (
|
|||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/request"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/playback"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/playback/utils"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
|
@ -55,26 +56,41 @@ func init() {
|
|||
runCmd.Flags().String(endpointFlag, "", "endpoint URL")
|
||||
}
|
||||
|
||||
func logResponse(cmd *cobra.Command, id int, resp *http.Response, logReq request.LoggedRequest) {
|
||||
func logResponse(cmd *cobra.Command, id int, resp *http.Response, logReq playback.LoggedRequest) {
|
||||
cmd.Println(strconv.Itoa(id)+")", logReq.Method, logReq.URI)
|
||||
cmd.Println(resp.Status)
|
||||
if resp.ContentLength != 0 {
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, resp.ContentLength))
|
||||
body := &bytes.Buffer{}
|
||||
isXML, checkBuf, err := utils.DetectXML(resp.Body)
|
||||
if err != nil {
|
||||
cmd.Println(id, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
resultWriter := utils.ChooseWriter(isXML, body)
|
||||
if _, err = resultWriter.Write(checkBuf); err != nil {
|
||||
cmd.Println(id, err)
|
||||
return
|
||||
}
|
||||
|
||||
_, err = io.Copy(resultWriter, io.LimitReader(resp.Body, viper.GetInt64(printResponseLimit)))
|
||||
if err != nil {
|
||||
cmd.Println(id, err)
|
||||
return
|
||||
}
|
||||
cmd.Println(string(body))
|
||||
|
||||
cmd.Println(body.String())
|
||||
}
|
||||
cmd.Println()
|
||||
}
|
||||
|
||||
func run(cmd *cobra.Command, _ []string) error {
|
||||
ctx := request.SetCredentials(
|
||||
ctx := playback.SetCredentials(
|
||||
cmd.Context(),
|
||||
viper.GetString(awsAccessKey),
|
||||
viper.GetString(awsSecretKey),
|
||||
)
|
||||
ctx = request.WithMultiparts(ctx)
|
||||
ctx = playback.WithMultiparts(ctx)
|
||||
|
||||
file, err := os.Open(viper.GetString(logPathFlag))
|
||||
if err != nil {
|
||||
|
|
@ -109,7 +125,7 @@ func run(cmd *cobra.Command, _ []string) error {
|
|||
case <-ctx.Done():
|
||||
return fmt.Errorf("interrupted: %w", ctx.Err())
|
||||
default:
|
||||
resp, err := playback(ctx, logReq, client)
|
||||
resp, err := playbackRequest(ctx, logReq, client)
|
||||
if err != nil {
|
||||
cmd.PrintErrln(strconv.Itoa(id)+")", "failed to playback request:", err)
|
||||
id++
|
||||
|
|
@ -123,8 +139,8 @@ func run(cmd *cobra.Command, _ []string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func getRequestFromLog(reader *bufio.Reader) (request.LoggedRequest, error) {
|
||||
var logReq request.LoggedRequest
|
||||
func getRequestFromLog(reader *bufio.Reader) (playback.LoggedRequest, error) {
|
||||
var logReq playback.LoggedRequest
|
||||
req, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return logReq, err
|
||||
|
|
@ -138,8 +154,8 @@ func getRequestFromLog(reader *bufio.Reader) (request.LoggedRequest, error) {
|
|||
return logReq, nil
|
||||
}
|
||||
|
||||
// playback creates http.Request from LoggedRequest and sends it to specified endpoint.
|
||||
func playback(ctx context.Context, logReq request.LoggedRequest, client *http.Client) (*http.Response, error) {
|
||||
// playbackRequest creates http.Request from LoggedRequest and sends it to specified endpoint.
|
||||
func playbackRequest(ctx context.Context, logReq playback.LoggedRequest, client *http.Client) (*http.Response, error) {
|
||||
r, err := prepareRequest(ctx, logReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to prepare request: %w", err)
|
||||
|
|
@ -155,7 +171,7 @@ func playback(ctx context.Context, logReq request.LoggedRequest, client *http.Cl
|
|||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if err = request.HandleResponse(ctx, r, respBody, logReq.Response); err != nil {
|
||||
if err = playback.HandleResponse(ctx, r, respBody, logReq.Response); err != nil {
|
||||
return nil, fmt.Errorf("failed to register multipart upload: %w", err)
|
||||
}
|
||||
resp.Body = io.NopCloser(bytes.NewBuffer(respBody))
|
||||
|
|
@ -164,29 +180,29 @@ func playback(ctx context.Context, logReq request.LoggedRequest, client *http.Cl
|
|||
}
|
||||
|
||||
// prepareRequest creates request from logs and modifies its signature and uploadId (if presents).
|
||||
func prepareRequest(ctx context.Context, logReq request.LoggedRequest) (*http.Request, error) {
|
||||
func prepareRequest(ctx context.Context, logReq playback.LoggedRequest) (*http.Request, error) {
|
||||
r, err := http.NewRequestWithContext(ctx, logReq.Method, viper.GetString(endpointFlag)+logReq.URI,
|
||||
bytes.NewReader(logReq.Body))
|
||||
bytes.NewReader(logReq.Payload))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.Header = logReq.Header
|
||||
|
||||
err = request.SwapUploadID(ctx, r)
|
||||
err = playback.SwapUploadID(ctx, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sha256hash := sha256.New()
|
||||
sha256hash.Write(logReq.Body)
|
||||
sha256hash.Write(logReq.Payload)
|
||||
r.Header.Set(auth.AmzContentSHA256, hex.EncodeToString(sha256hash.Sum(nil)))
|
||||
if r.Header.Get(api.ContentMD5) != "" {
|
||||
sha256hash.Reset()
|
||||
md5hash := md5.New()
|
||||
md5hash.Write(logReq.Body)
|
||||
md5hash.Write(logReq.Payload)
|
||||
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(md5hash.Sum(nil)))
|
||||
}
|
||||
err = request.Sign(ctx, r)
|
||||
err = playback.Sign(ctx, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
endpoint: http://localhost:8084
|
||||
log: ./log/multipart5.log
|
||||
log: ./log/request.log
|
||||
credentials:
|
||||
access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD
|
||||
secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30
|
||||
http_timeout: 60s
|
||||
skip_verify_tls: false
|
||||
print_response_limit: 1024
|
||||
|
|
|
|||
|
|
@ -389,7 +389,6 @@ http_logging:
|
|||
max_log_size: 20
|
||||
gzip: true
|
||||
destination: stdout
|
||||
log_response: true
|
||||
```
|
||||
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|
|
@ -399,7 +398,6 @@ http_logging:
|
|||
| `max_log_size` | int | yes | 50 | Log file size threshold (in megabytes) to be moved in backup file. After reaching threshold, initial filename is appended with timestamp. And new empty file with initial name is created. |
|
||||
| `gzip` | bool | yes | false | Whether to enable Gzip compression to backup log files. |
|
||||
| `destination` | string | yes | stdout | Specify path for log output. Accepts log file path, or "stdout" and "stderr" reserved words to print in output streams. File and folders are created if necessary. |
|
||||
| `log_response` | bool | yes | true | Whether to attach response body to the log. |
|
||||
|
||||
|
||||
### `cache` section
|
||||
|
|
|
|||
|
|
@ -31,14 +31,15 @@ If corresponding flag is set, it overrides parameter from config.
|
|||
|
||||
### Configuration parameters
|
||||
|
||||
#### Global params
|
||||
| # | Config parameter name | Flag name | Type | Default value | Description |
|
||||
|---|-----------------------|-----------------|----------|---------------|--------------------------------------------------------|
|
||||
| 1 | - | config | string | - | config file path (e.g. `./config/playback.yaml`) |
|
||||
| 2 | http_timeout | http-timeout | duration | 60s | http request timeout |
|
||||
| 3 | skip_verify_tls | skip-verify-tls | bool | false | skips tls certificate verification for https endpoints |
|
||||
| 4 | credentials.accessKey | - | string | - | AWS access key id |
|
||||
| 5 | credentials.secretKey | - | string | - | AWS secret key |
|
||||
#### Global parameters
|
||||
| # | Config parameter name | Flag name | Type | Default value | Description |
|
||||
|---|-----------------------|-----------------|----------|---------------|-------------------------------------------------------------------------------|
|
||||
| 1 | - | config | string | - | config file path (e.g. `./config/playback.yaml`) |
|
||||
| 2 | http_timeout | http-timeout | duration | 60s | http request timeout |
|
||||
| 3 | skip_verify_tls | skip-verify-tls | bool | false | skips tls certificate verification for https endpoints |
|
||||
| 4 | credentials.accessKey | - | string | - | AWS access key id |
|
||||
| 5 | credentials.secretKey | - | string | - | AWS secret key |
|
||||
| 6 | print_response_limit | - | int | 1024 | max response length to be printed in stdout, the rest of body will be omitted |
|
||||
|
||||
#### `run` command parameters
|
||||
| # | Config parameter name | Flag name | Type | Default value | Description |
|
||||
|
|
|
|||
|
|
@ -3,14 +3,17 @@ package playback
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/playback/utils"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
)
|
||||
|
|
@ -22,14 +25,15 @@ var (
|
|||
)
|
||||
|
||||
type (
|
||||
httpBody []byte
|
||||
LoggedRequest struct {
|
||||
From string `json:"from"`
|
||||
URI string `json:"URI"`
|
||||
Method string `json:"method"`
|
||||
Payload httpBody `json:"payload,omitempty"`
|
||||
Response httpBody `json:"response,omitempty"`
|
||||
Query url.Values `json:"query"`
|
||||
Body []byte `json:"body"`
|
||||
Header http.Header `json:"headers"`
|
||||
Response []byte `json:"response"`
|
||||
}
|
||||
Credentials struct {
|
||||
AccessKey string
|
||||
|
|
@ -39,6 +43,21 @@ type (
|
|||
multipartKey struct{}
|
||||
)
|
||||
|
||||
func (h *httpBody) UnmarshalJSON(data []byte) error {
|
||||
unquoted, err := strconv.Unquote(string(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
isXML, _, err := utils.DetectXML(strings.NewReader(unquoted))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reader := utils.ChooseReader(isXML, strings.NewReader(unquoted))
|
||||
*h, err = io.ReadAll(reader)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func SetCredentials(ctx context.Context, accessKey, secretKey string) context.Context {
|
||||
return context.WithValue(ctx, contextKey{},
|
||||
Credentials{
|
||||
|
|
|
|||
|
|
@ -35,8 +35,8 @@ func ChooseWriter(isXML bool, body *bytes.Buffer) io.Writer {
|
|||
return body
|
||||
}
|
||||
|
||||
func ChooseReader(isXml bool, bodyReader io.Reader) io.Reader {
|
||||
if !isXml {
|
||||
func ChooseReader(isXML bool, bodyReader io.Reader) io.Reader {
|
||||
if !isXML {
|
||||
return base64.NewDecoder(base64.StdEncoding, bodyReader)
|
||||
}
|
||||
return bodyReader
|
||||
|
|
|
|||
Loading…
Reference in a new issue