[#369] Request reproducer #443

Merged
nzinkevich merged 2 commits from nzinkevich/frostfs-s3-gw:feature/playback into feature/369 2024-09-04 19:51:14 +00:00
13 changed files with 624 additions and 9 deletions

View file

@ -3,11 +3,12 @@ FROM golang:1.21 as builder
ARG BUILD=now
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
ARG VERSION=dev
ARG GOFLAGS=""
WORKDIR /src
COPY . /src
RUN make
RUN make GOFLAGS=${GOFLAGS}
# Executable image
FROM alpine AS frostfs-s3-gw

View file

@ -14,6 +14,8 @@ METRICS_DUMP_OUT ?= ./metrics-dump.json
CMDS = $(addprefix frostfs-, $(notdir $(wildcard cmd/*)))
BINS = $(addprefix $(BINDIR)/, $(CMDS))
GOFLAGS ?=
# Variables for docker
REPO_BASENAME = $(shell basename `go list -m`)
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
@ -38,6 +40,7 @@ all: $(BINS)
$(BINS): $(BINDIR) dep
@echo "⇒ Build $@"
CGO_ENABLED=0 \
GOFLAGS=$(GOFLAGS) \
go build -v -trimpath \
-ldflags "-X $(REPO)/internal/version.Version=$(VERSION)" \
-o $@ ./cmd/$(subst frostfs-,,$(notdir $@))
@ -64,7 +67,7 @@ docker/%:
-w /src \
-u `stat -c "%u:%g" .` \
--env HOME=/src \
golang:$(GO_VERSION) make $*,\
golang:$(GO_VERSION) make GOFLAGS=$(GOFLAGS) $*,\
@echo "supported docker targets: all $(BINS) lint")
# Run tests
@ -87,6 +90,7 @@ image:
@docker build \
--build-arg REPO=$(REPO) \
--build-arg VERSION=$(VERSION) \
--build-arg GOFLAGS=$(GOFLAGS) \
--rm \
-f .docker/Dockerfile \
-t $(HUB_IMAGE):$(HUB_TAG) .

View file

@ -0,0 +1,7 @@
endpoint: http://localhost:8084
log: ./log/multipart5.log
credentials:
access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD
secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30
http_timeout: 60s
skip_verify_tls: false

20
cmd/s3-playback/main.go Normal file
View file

@ -0,0 +1,20 @@
package main
import (
"context"
"os"
"os/signal"
"syscall"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/modules"
)
func main() {
ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
if cmd, err := modules.Execute(ctx); err != nil {
cmd.PrintErrln("Error:", err.Error())
cmd.PrintErrf("Run '%v --help' for usage.\n", cmd.CommandPath())
os.Exit(1)
}
}
Outdated
Review

blank line

blank line

View file

@ -0,0 +1,65 @@
package modules
import (
"context"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
const (
configPath = "config"
configPathFlag = "config"
httpTimeout = "http_timeout"
httpTimeoutFlag = "http-timeout"
skipVerifyTLS = "skip_verify_tls"
skipVerifyTLSFlag = "skip-verify-tls"
)
var (
cfgFile string
rootCmd = &cobra.Command{
Use: "frostfs-s3-playback",
Version: version.Version,
Short: "FrostFS S3 Traffic Playback",
Long: "Helps to reproduce s3 commands from log files",
Example: "frostfs-s3-playback [--skip-verify-tls] [--http-timeout <timeout>] " +
"[--version] --config <config_path> <command>",
SilenceUsage: true,
SilenceErrors: true,
PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
_ = viper.BindPFlag(configPath, cmd.PersistentFlags().Lookup(configPathFlag))
_ = viper.BindPFlag(httpTimeout, cmd.PersistentFlags().Lookup(httpTimeoutFlag))
_ = viper.BindPFlag(skipVerifyTLS, cmd.PersistentFlags().Lookup(skipVerifyTLSFlag))
return nil
},
RunE: func(cmd *cobra.Command, _ []string) error {
return cmd.Help()
},
}

Let's keep statement in one line

Let's keep statement in one line
)
func Execute(ctx context.Context) (*cobra.Command, error) {
return rootCmd.ExecuteContextC(ctx)
}
Outdated
Review

blank line

blank line
rootCmd.PersistentFlags().Duration(httpTimeoutFlag, time.Minute, "http request timeout")
```golang rootCmd.PersistentFlags().Duration(httpTimeoutFlag, time.Minute, "http request timeout") ````
func initConfig() {
viper.SetConfigFile(cfgFile)
_ = viper.ReadInConfig()
}

Maybe we can move flag binding into PersistentPreRunE of rootCmd?

Maybe we can move flag binding into `PersistentPreRunE` of `rootCmd`?
func init() {
cobra.OnInitialize(initConfig)
rootCmd.PersistentFlags().StringVar(&cfgFile, configPathFlag, "", "configuration filepath")
_ = rootCmd.MarkPersistentFlagRequired(configPathFlag)
_ = rootCmd.MarkPersistentFlagFilename(configPathFlag)
rootCmd.PersistentFlags().Duration(httpTimeoutFlag, time.Minute, "http request timeout")
rootCmd.PersistentFlags().Bool(skipVerifyTLSFlag, false, "skip TLS certificate verification")
rootCmd.AddCommand(runCmd)
}

View file

@ -0,0 +1,194 @@
package modules
import (
"bufio"
"bytes"
"context"
"crypto/md5"
"crypto/sha256"
"crypto/tls"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"strconv"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/request"
alexvanin marked this conversation as resolved Outdated

In case of package name collision, try to provide context to an alias name instead of putting 2 in the end.

In this case we can name it s3request.

In case of package name collision, try to provide context to an alias name instead of putting 2 in the end. In this case we can name it `s3request`.
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
const (
logPathFlag = "log"
endpointFlag = "endpoint"
dkirillov marked this conversation as resolved Outdated

Please fill Example field

Please fill `Example` field
awsAccessKey = "credentials.access_key"
awsSecretKey = "credentials.secret_key"
)
var runCmd = &cobra.Command{
Use: "run",
Short: "Send requests from log file",
dkirillov marked this conversation as resolved Outdated

Try to not use global varialbe

Try to not use global varialbe
Long: "Reads the network log file and sends each request to the specified URL",
Example: "frostfs-s3-playback --config <config_path> run [--endpoint=<endpoint>] [--log=<log_path>]",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
dkirillov marked this conversation as resolved Outdated

It's better to use Println function from cmd *cobra.Command

It's better to use `Println` function from `cmd *cobra.Command`
if rootCmd.PersistentPreRunE != nil {
if err := rootCmd.PersistentPreRunE(cmd, args); err != nil {
return err
}
}
_ = viper.BindPFlag(logPathFlag, cmd.Flags().Lookup(logPathFlag))
_ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag))
return nil
},
RunE: run,
dkirillov marked this conversation as resolved Outdated

Let's move this near to the place where it's used. (e.g. before or after line for sc.Scan() {)

Let's move this near to the place where it's used. (e.g. before or after line `for sc.Scan() {`)
}
dkirillov marked this conversation as resolved Outdated

We can omit var:

creds := request.Credentials{/*...*/}
We can omit `var`: ``` creds := request.Credentials{/*...*/} ```
Outdated
Review

credentials.access_key & credentials.secret_key
should be constants

credentials.access_key & credentials.secret_key should be constants
func init() {
runCmd.Flags().String(logPathFlag, "./request.log", "log file path")
runCmd.Flags().String(endpointFlag, "", "endpoint URL")
dkirillov marked this conversation as resolved Outdated

We should use context from command.

func run (cmd *cobra.Command, _ []string) error {
    // ...
    ctx := context.WithValue(cmd.Context(), ...)
    // ...
}
We should use context from command. ``` func run (cmd *cobra.Command, _ []string) error { // ... ctx := context.WithValue(cmd.Context(), ...) // ... } ```
}
func logResponse(cmd *cobra.Command, id int, resp *http.Response, logReq request.LoggedRequest) {
cmd.Println(strconv.Itoa(id)+")", logReq.Method, logReq.URI)
cmd.Println(resp.Status)
if resp.ContentLength != 0 {
body, err := io.ReadAll(io.LimitReader(resp.Body, resp.ContentLength))
if err != nil {
cmd.Println(id, err)
}
cmd.Println(string(body))
}
dkirillov marked this conversation as resolved Outdated

I think this isn't necessary. We can create new map in SetMultipartUpload if it doesn't exist yet

I think this isn't necessary. We can create new map in `SetMultipartUpload` if it doesn't exist yet

But then SetMultipartUpload should also return an updated context or I should pass a *cobra.Command argument to it for SetContext(ctx). I think both ways look worse.

But then SetMultipartUpload should also return an updated context or I should pass a *cobra.Command argument to it for SetContext(ctx). I think both ways look worse.
cmd.Println()
dkirillov marked this conversation as resolved Outdated

Why do we do this?
It seems we can pass only context to playback function

Why do we do this? It seems we can pass only context to `playback` function
}
func run(cmd *cobra.Command, _ []string) error {
ctx := request.SetCredentials(
dkirillov marked this conversation as resolved Outdated

It's quite odd to set buffer that has size of full file to read. I believe we can skip setting this param at all (or we can configure this buffer size).

It's quite odd to set buffer that has size of full file to read. I believe we can skip setting this param at all (or we can configure this buffer size).
cmd.Context(),
viper.GetString(awsAccessKey),
viper.GetString(awsSecretKey),
)
dkirillov marked this conversation as resolved Outdated

Please, use cmd.Println

Please, use `cmd.Println`
ctx = request.WithMultiparts(ctx)
file, err := os.Open(viper.GetString(logPathFlag))
if err != nil {
return err

We can use viper.GetDuration

We can use `viper.GetDuration`
}
defer file.Close()
reader := bufio.NewReader(file)
client := &http.Client{
Transport: http.DefaultTransport,
Timeout: viper.GetDuration(httpTimeout),
dkirillov marked this conversation as resolved Outdated

It seems message should be failed to prepare request:

It seems message should be `failed to prepare request:`
}
if viper.GetBool(skipVerifyTLS) {
dkirillov marked this conversation as resolved Outdated

Don't use default client. It has no timeout so in theory it can hang out

Don't use default client. It has no timeout so in theory it can hang out
client.Transport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
id := 1
for {
logReq, err := getRequestFromLog(reader)
if err != nil {
if err == io.EOF {
break
}
cmd.PrintErrln(strconv.Itoa(id)+")", "failed to parse request", err)
id++
continue
}
select {
case <-ctx.Done():
return fmt.Errorf("interrupted: %w", ctx.Err())
dkirillov marked this conversation as resolved Outdated

Use:

r, err := request.NewRequest(viper.GetString(endpointFlag), data)
Use: ``` r, err := request.NewRequest(viper.GetString(endpointFlag), data) ```
default:
resp, err := playback(ctx, logReq, client)
if err != nil {
cmd.PrintErrln(strconv.Itoa(id)+")", "failed to playback request:", err)
id++
continue
}
logResponse(cmd, id, resp, logReq)

Why don't we invoke request.SwapUploadID and request.Sign inside request.NewRequest?

Why don't we invoke `request.SwapUploadID` and `request.Sign` inside `request.NewRequest`?

request.NewRequest was removed, so considered to leave these functions in the prepareRequest

request.NewRequest was removed, so considered to leave these functions in the prepareRequest
id++
}
}
return nil
Outdated
Review

blank line

blank line
}
func getRequestFromLog(reader *bufio.Reader) (request.LoggedRequest, error) {
var logReq request.LoggedRequest
req, err := reader.ReadString('\n')
if err != nil {
return logReq, err
}
err = json.Unmarshal([]byte(req), &logReq)
if err != nil {
return logReq, err
}
return logReq, nil
}
// playback creates http.Request from LoggedRequest and sends it to specified endpoint.
func playback(ctx context.Context, logReq request.LoggedRequest, client *http.Client) (*http.Response, error) {
r, err := prepareRequest(ctx, logReq)
if err != nil {
return nil, fmt.Errorf("failed to prepare request: %w", err)
}
resp, err := client.Do(r)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}
alexvanin marked this conversation as resolved Outdated

We definitely should use http.NewRequestWithContext here. Network request may take some time, we don't want to wait in case of SIGTERM or any other context cancellation.

We definitely should use `http.NewRequestWithContext` here. Network request may take some time, we don't want to wait in case of `SIGTERM` or any other context cancellation.
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body: %w", err)
}
defer resp.Body.Close()
if err = request.HandleResponse(ctx, r, respBody, logReq.Response); err != nil {
return nil, fmt.Errorf("failed to register multipart upload: %w", err)
}
resp.Body = io.NopCloser(bytes.NewBuffer(respBody))
return resp, nil
}
// prepareRequest creates request from logs and modifies its signature and uploadId (if presents).
func prepareRequest(ctx context.Context, logReq request.LoggedRequest) (*http.Request, error) {
r, err := http.NewRequestWithContext(ctx, logReq.Method, viper.GetString(endpointFlag)+logReq.URI,
bytes.NewReader(logReq.Body))
if err != nil {
return nil, err
}
r.Header = logReq.Header
err = request.SwapUploadID(ctx, r)
if err != nil {
return nil, err
}
sha256hash := sha256.New()
sha256hash.Write(logReq.Body)
r.Header.Set(auth.AmzContentSHA256, hex.EncodeToString(sha256hash.Sum(nil)))
if r.Header.Get(api.ContentMD5) != "" {
sha256hash.Reset()
md5hash := md5.New()
md5hash.Write(logReq.Body)
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(md5hash.Sum(nil)))
}
err = request.Sign(ctx, r)
if err != nil {
return nil, err
}
return r, nil
}

View file

@ -0,0 +1,74 @@
package request
import (
"context"
"encoding/xml"
"errors"
"fmt"
"net/http"
)
type MultipartUpload struct {
dkirillov marked this conversation as resolved Outdated

Try to not use global varialbe

Try to not use global varialbe
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
Bucket string `json:"bucket" xml:"Bucket"`
Key string `json:"key" xml:"Key"`
UploadID string `json:"uploadId" xml:"UploadId"`
}
func WithMultiparts(ctx context.Context) context.Context {
return context.WithValue(ctx, multipartKey{}, map[string]MultipartUpload{})
}
dkirillov marked this conversation as resolved Outdated

It's better to rename this function to something like HandleResponse

It's better to rename this function to something like `HandleResponse`
func SetMultipartUpload(ctx context.Context, oldUploadID string, upload MultipartUpload) error {
mparts, ok := ctx.Value(multipartKey{}).(map[string]MultipartUpload)
if !ok {
return errors.New("multiparts not set")
}
mparts[oldUploadID] = upload
return nil
}
func GetMultipart(ctx context.Context, oldUploadID string) (MultipartUpload, error) {
mparts, ok := ctx.Value(multipartKey{}).(map[string]MultipartUpload)
if !ok {
return MultipartUpload{}, errors.New("no multipart map set")
}
return mparts[oldUploadID], nil
}
func HandleResponse(ctx context.Context, r *http.Request, resp []byte, logResponse []byte) error {
var mpart, mpartOld MultipartUpload
if r.Method == "POST" && r.URL.Query().Has("uploads") {
err1 := xml.Unmarshal(resp, &mpart)
err2 := xml.Unmarshal(logResponse, &mpartOld)
if err1 != nil || err2 != nil {
return errors.New("xml unmarshal error")
}
if mpartOld.UploadID != "" {
if err := SetMultipartUpload(ctx, mpartOld.UploadID, mpart); err != nil {
return err
}
Outdated
Review

blank line

blank line
}
}
return nil
}
func SwapUploadID(ctx context.Context, r *http.Request) error {
var uploadID string
query := r.URL.Query()
if query.Has("uploadId") {
uploadID = query.Get("uploadId")
mpart, err := GetMultipart(ctx, uploadID)
if err != nil {
return fmt.Errorf("failed to get multipart upload: %w", err)
}
query.Set("uploadId", mpart.UploadID)
r.URL.RawQuery = query.Encode()
}
return nil
}

View file

@ -0,0 +1,101 @@
package request
import (
"context"
"errors"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/credentials"
Outdated
Review

we have own signer v4

we have own [signer](https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/src/branch/master/api/auth/signer/v4/v4.go#L167) v4

I chose AWS SDK v2 because it has the Sign HTTP method, which is convenient for updating the S3 request signature. SDK v2 is already set in the go.mod file, but I also added the "github.com/aws/aws-sdk-go-v2/aws/credentials" package. @dkirillov

I chose AWS SDK v2 because it has the Sign HTTP method, which is convenient for updating the S3 request signature. SDK v2 is already set in the go.mod file, but I also added the "github.com/aws/aws-sdk-go-v2/aws/credentials" package. @dkirillov
)
var (
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
Outdated
Review

no usages

no usages
alexvanin marked this conversation as resolved Outdated

We need some unit-tests to validate this regexp, just in case.

We need some unit-tests to validate this regexp, just in case.

Maybe we can also reuse this regexp from api/auth/center.go

Maybe we can also reuse this regexp from `api/auth/center.go`
ErrNoMatches = errors.New("no matches found")
)
type (
LoggedRequest struct {
From string `json:"from"`
URI string `json:"URI"`
Method string `json:"method"`
Query url.Values `json:"query"`
Body []byte `json:"body"`
Header http.Header `json:"headers"`
Response []byte `json:"response"`
}
Credentials struct {
AccessKey string
SecretKey string
dkirillov marked this conversation as resolved Outdated

Let's write something like

type credKeyType struct{}

func SetCredentials(ctx context.Context, creds Credentials) context.Context {
	return context.WithValue(ctx, credKeyType{}, creds)
}

func GetCredentials(ctx context.Context) (Credentials, error) {
	creds, ok := ctx.Value(credKeyType{}).(Credentials)
	if !ok {
		return Credentials{}, errors.New("credentials is missing in contex")
	}

	return creds, nil
}
Let's write something like ``` type credKeyType struct{} func SetCredentials(ctx context.Context, creds Credentials) context.Context { return context.WithValue(ctx, credKeyType{}, creds) } func GetCredentials(ctx context.Context) (Credentials, error) { creds, ok := ctx.Value(credKeyType{}).(Credentials) if !ok { return Credentials{}, errors.New("credentials is missing in contex") } return creds, nil } ```
}
contextKey struct{}
multipartKey struct{}
)
func SetCredentials(ctx context.Context, accessKey, secretKey string) context.Context {
return context.WithValue(ctx, contextKey{},
Credentials{
AccessKey: accessKey, SecretKey: secretKey,
},
)
}
func GetCredentials(ctx context.Context) (Credentials, error) {
val, ok := ctx.Value(contextKey{}).(Credentials)
if !ok {
return val, errors.New("credentials not set")
}
return val, nil
}
// Sign replace Authorization header with new Access key id and Signature values.
func Sign(ctx context.Context, r *http.Request) error {
creds, err := GetCredentials(ctx)
if err != nil {
return errors.New("failed to get credentials")
}
credProvider, err := credentials.NewStaticCredentialsProvider(creds.AccessKey,
creds.SecretKey, "").Retrieve(ctx)
if err != nil {
return err
}
authInfo, err := parseAuthHeader(r.Header.Get(auth.AuthorizationHdr))
if err != nil {
return err
}
r.Header[auth.AuthorizationHdr][0] = strings.Replace(r.Header[auth.AuthorizationHdr][0],
authInfo["access_key_id"], creds.AccessKey, 1)
signer := v4.NewSigner()
signatureDateTimeStr := r.Header.Get(api.AmzDate)
signatureDateTime, err := time.Parse("20060102T150405Z", signatureDateTimeStr)
if err != nil {
return err
}
err = signer.SignHTTP(ctx, credProvider, r, r.Header.Get(api.AmzContentSha256),
authInfo["service"], authInfo["region"], signatureDateTime)
if err != nil {
return err
}
return nil
}
dkirillov marked this conversation as resolved Outdated
Outdated
Review

do we need this?

do we need this?

I will use this later to replace the 'X-Amz-Content-SHA256' header. This is useful to ensure that the hash relates to the actual body of the request, which can sometimes be incomplete when saved in logs.

I will use this later to replace the 'X-Amz-Content-SHA256' header. This is useful to ensure that the hash relates to the actual body of the request, which can sometimes be incomplete when saved in logs.

Then we should add this function when we will actually use it rather than keep commented code.

Then we should add this function when we will actually use it rather than keep commented code.
func parseAuthHeader(authHeader string) (map[string]string, error) {
authInfo := auth.NewRegexpMatcher(authorizationFieldRegexp).GetSubmatches(authHeader)
if len(authInfo) == 0 {
return nil, ErrNoMatches
}
return authInfo, nil
}

View file

@ -0,0 +1,95 @@
package request
import (
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func withoutValue(data map[string]string, field string) map[string]string {
result := make(map[string]string)
for k, v := range data {
result[k] = v
}
result[field] = ""
return result
}
func TestParseAuthHeader(t *testing.T) {
defaultHeader := "AWS4-HMAC-SHA256 Credential=oid0cid/20210809/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2811ccb9e242f41426738fb1f"
defaultAuthInfo := map[string]string{
"access_key_id": "oid0cid",
"service": "s3",
"region": "us-east-1",
"v4_signature": "2811ccb9e242f41426738fb1f",
"signed_header_fields": "host;x-amz-content-sha256;x-amz-date",
"date": "20210809",
}
for _, tc := range []struct {
title string
header string
err error
expected map[string]string
}{
{
title: "correct full header",
header: defaultHeader,
err: nil,
expected: defaultAuthInfo,
},
{
title: "correct with empty region",
header: strings.Replace(defaultHeader, "/us-east-1/", "//", 1),
err: nil,
expected: withoutValue(defaultAuthInfo, "region"),
},
{
title: "empty access key",
header: strings.Replace(defaultHeader, "oid0cid", "", 1),
err: ErrNoMatches,
expected: nil,
},
{
title: "empty service",
header: strings.Replace(defaultHeader, "/s3/", "//", 1),
err: ErrNoMatches,
expected: nil,
},
{
title: "empty date",
header: strings.Replace(defaultHeader, "/20210809/", "//", 1),
err: ErrNoMatches,
expected: nil,
},
{
title: "empty v4_signature",
header: strings.Replace(defaultHeader, "Signature=2811ccb9e242f41426738fb1f",
"Signature=", 1),
err: ErrNoMatches,
expected: nil,
},
{
title: "empty signed_fields",
header: strings.Replace(defaultHeader, "SignedHeaders=host;x-amz-content-sha256;x-amz-date",
"SignedHeaders=", 1),
err: ErrNoMatches,
expected: nil,
},
{
title: "empty signed_fields",
header: strings.Replace(defaultHeader, "SignedHeaders=host;x-amz-content-sha256;x-amz-date",
"SignedHeaders=", 1),
err: ErrNoMatches,
expected: nil,
},
} {
t.Run(tc.title, func(t *testing.T) {
authInfo, err := parseAuthHeader(tc.header)
require.ErrorIs(t, err, tc.err, tc.header)
require.Equal(t, tc.expected, authInfo, tc.header)
})
}
}

View file

@ -377,6 +377,11 @@ logger:
### `http_logging` section
Could be enabled only in builds with `loghttp` build tag. To build with `loghttp` tag, pass `GOFLAGS` var to `make`:
```bash
make GOFLAGS="-tags=loghttp" [target]

Needs to be updated with new make command.

Needs to be updated with new make command.
```
```yaml
http_logging:
enabled: false

47
docs/playback.md Normal file
View file

@ -0,0 +1,47 @@
# FrostFS S3 Playback
Playback is a tool to reproduce queries to `frostfs-s3-gw` in dev environment. Network logs could be
gathered from `s3-gw` via HTTP Logger which could be enabled on build with `loghttp` build tag
and `http_logging.enabled` option set to `true` in `s3-gw` configuration.
## Commands
`run` - reads log file and reproduces send requests from it to specified endpoint
#### Example
```bash
frostfs-s3-playback --config <config_path> run [--endpoint=<endpoint>] [--log=<log_path>]
```
## Configuration
Playback accepts configuration file path in yaml with corresponding options:
```yaml
endpoint: http://localhost:8084
log: ./request.log
env: .env
credentials:
access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD
secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30
http_timeout: 60s
skip_verify_tls: true
```
Configuration path is passed via required `--config` flag.
If corresponding flag is set, it overrides parameter from config.
### Configuration parameters
#### Global params
| # | Config parameter name | Flag name | Type | Default value | Description |
|---|-----------------------|-----------------|----------|---------------|--------------------------------------------------------|
| 1 | - | config | string | - | config file path (e.g. `./config/playback.yaml`) |
| 2 | http_timeout | http-timeout | duration | 60s | http request timeout |
| 3 | skip_verify_tls | skip-verify-tls | bool | false | skips tls certificate verification for https endpoints |
| 4 | credentials.accessKey | - | string | - | AWS access key id |
| 5 | credentials.secretKey | - | string | - | AWS secret key |
#### `run` command parameters
| # | Config parameter name | Flag name | Type | Default value | Description |
|---|-----------------------|-----------|--------|---------------|--------------------------------------------------------|
| 1 | endpoint | endpoint | string | - | s3-gw endpoint URL |
| 2 | log | log | string | ./request.log | path to log file, could be either absolute or relative |

5
go.mod
View file

@ -10,7 +10,8 @@ require (
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
github.com/aws/aws-sdk-go v1.44.6
github.com/aws/aws-sdk-go-v2 v1.18.1
github.com/aws/aws-sdk-go-v2 v1.30.3
github.com/aws/aws-sdk-go-v2/credentials v1.17.27
github.com/bluele/gcache v0.0.2
github.com/go-chi/chi/v5 v5.0.8
github.com/google/uuid v1.3.1
@ -43,7 +44,7 @@ require (
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/aws/smithy-go v1.20.3 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect

11
go.sum
View file

@ -64,10 +64,12 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
github.com/aws/aws-sdk-go v1.44.6 h1:Y+uHxmZfhRTLX2X3khkdxCoTZAyGEX21aOUHe1U6geg=
github.com/aws/aws-sdk-go v1.44.6/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo=
github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI=
github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
@ -172,7 +174,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=