[#369] Request reproducer #443
|
@ -3,11 +3,12 @@ FROM golang:1.21 as builder
|
|||
ARG BUILD=now
|
||||
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
||||
ARG VERSION=dev
|
||||
ARG GOFLAGS=""
|
||||
|
||||
WORKDIR /src
|
||||
COPY . /src
|
||||
|
||||
RUN make
|
||||
RUN make GOFLAGS=${GOFLAGS}
|
||||
|
||||
# Executable image
|
||||
FROM alpine AS frostfs-s3-gw
|
||||
|
|
6
Makefile
|
@ -14,6 +14,8 @@ METRICS_DUMP_OUT ?= ./metrics-dump.json
|
|||
CMDS = $(addprefix frostfs-, $(notdir $(wildcard cmd/*)))
|
||||
BINS = $(addprefix $(BINDIR)/, $(CMDS))
|
||||
|
||||
GOFLAGS ?=
|
||||
|
||||
# Variables for docker
|
||||
REPO_BASENAME = $(shell basename `go list -m`)
|
||||
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
|
||||
|
@ -38,6 +40,7 @@ all: $(BINS)
|
|||
$(BINS): $(BINDIR) dep
|
||||
@echo "⇒ Build $@"
|
||||
CGO_ENABLED=0 \
|
||||
GOFLAGS=$(GOFLAGS) \
|
||||
go build -v -trimpath \
|
||||
-ldflags "-X $(REPO)/internal/version.Version=$(VERSION)" \
|
||||
-o $@ ./cmd/$(subst frostfs-,,$(notdir $@))
|
||||
|
@ -64,7 +67,7 @@ docker/%:
|
|||
-w /src \
|
||||
-u `stat -c "%u:%g" .` \
|
||||
--env HOME=/src \
|
||||
golang:$(GO_VERSION) make $*,\
|
||||
golang:$(GO_VERSION) make GOFLAGS=$(GOFLAGS) $*,\
|
||||
@echo "supported docker targets: all $(BINS) lint")
|
||||
|
||||
# Run tests
|
||||
|
@ -87,6 +90,7 @@ image:
|
|||
@docker build \
|
||||
--build-arg REPO=$(REPO) \
|
||||
--build-arg VERSION=$(VERSION) \
|
||||
--build-arg GOFLAGS=$(GOFLAGS) \
|
||||
--rm \
|
||||
-f .docker/Dockerfile \
|
||||
-t $(HUB_IMAGE):$(HUB_TAG) .
|
||||
|
|
7
cmd/s3-playback/config/playback.yaml
Normal file
|
@ -0,0 +1,7 @@
|
|||
endpoint: http://localhost:8084
|
||||
log: ./log/multipart5.log
|
||||
credentials:
|
||||
access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD
|
||||
secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30
|
||||
http_timeout: 60s
|
||||
skip_verify_tls: false
|
20
cmd/s3-playback/main.go
Normal file
|
@ -0,0 +1,20 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/modules"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
|
||||
|
||||
if cmd, err := modules.Execute(ctx); err != nil {
|
||||
cmd.PrintErrln("Error:", err.Error())
|
||||
cmd.PrintErrf("Run '%v --help' for usage.\n", cmd.CommandPath())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
65
cmd/s3-playback/modules/root.go
Normal file
|
@ -0,0 +1,65 @@
|
|||
package modules
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
const (
|
||||
configPath = "config"
|
||||
configPathFlag = "config"
|
||||
httpTimeout = "http_timeout"
|
||||
httpTimeoutFlag = "http-timeout"
|
||||
skipVerifyTLS = "skip_verify_tls"
|
||||
skipVerifyTLSFlag = "skip-verify-tls"
|
||||
)
|
||||
|
||||
var (
|
||||
cfgFile string
|
||||
rootCmd = &cobra.Command{
|
||||
Use: "frostfs-s3-playback",
|
||||
Version: version.Version,
|
||||
Short: "FrostFS S3 Traffic Playback",
|
||||
Long: "Helps to reproduce s3 commands from log files",
|
||||
Example: "frostfs-s3-playback [--skip-verify-tls] [--http-timeout <timeout>] " +
|
||||
"[--version] --config <config_path> <command>",
|
||||
SilenceUsage: true,
|
||||
SilenceErrors: true,
|
||||
PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
|
||||
_ = viper.BindPFlag(configPath, cmd.PersistentFlags().Lookup(configPathFlag))
|
||||
|
||||
_ = viper.BindPFlag(httpTimeout, cmd.PersistentFlags().Lookup(httpTimeoutFlag))
|
||||
|
||||
_ = viper.BindPFlag(skipVerifyTLS, cmd.PersistentFlags().Lookup(skipVerifyTLSFlag))
|
||||
|
||||
return nil
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
return cmd.Help()
|
||||
},
|
||||
}
|
||||
dkirillov
commented
Let's keep statement in one line Let's keep statement in one line
|
||||
)
|
||||
|
||||
func Execute(ctx context.Context) (*cobra.Command, error) {
|
||||
return rootCmd.ExecuteContextC(ctx)
|
||||
}
|
||||
pogpp
commented
blank line blank line
dkirillov
commented
```golang
rootCmd.PersistentFlags().Duration(httpTimeoutFlag, time.Minute, "http request timeout")
````
|
||||
|
||||
func initConfig() {
|
||||
viper.SetConfigFile(cfgFile)
|
||||
_ = viper.ReadInConfig()
|
||||
}
|
||||
|
||||
dkirillov
commented
Maybe we can move flag binding into Maybe we can move flag binding into `PersistentPreRunE` of `rootCmd`?
|
||||
func init() {
|
||||
cobra.OnInitialize(initConfig)
|
||||
rootCmd.PersistentFlags().StringVar(&cfgFile, configPathFlag, "", "configuration filepath")
|
||||
_ = rootCmd.MarkPersistentFlagRequired(configPathFlag)
|
||||
_ = rootCmd.MarkPersistentFlagFilename(configPathFlag)
|
||||
rootCmd.PersistentFlags().Duration(httpTimeoutFlag, time.Minute, "http request timeout")
|
||||
rootCmd.PersistentFlags().Bool(skipVerifyTLSFlag, false, "skip TLS certificate verification")
|
||||
|
||||
rootCmd.AddCommand(runCmd)
|
||||
}
|
194
cmd/s3-playback/modules/run.go
Normal file
|
@ -0,0 +1,194 @@
|
|||
package modules
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/request"
|
||||
alexvanin marked this conversation as resolved
Outdated
alexvanin
commented
In case of package name collision, try to provide context to an alias name instead of putting 2 in the end. In this case we can name it In case of package name collision, try to provide context to an alias name instead of putting 2 in the end.
In this case we can name it `s3request`.
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
const (
|
||||
logPathFlag = "log"
|
||||
endpointFlag = "endpoint"
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
Please fill Please fill `Example` field
|
||||
awsAccessKey = "credentials.access_key"
|
||||
awsSecretKey = "credentials.secret_key"
|
||||
)
|
||||
|
||||
var runCmd = &cobra.Command{
|
||||
Use: "run",
|
||||
Short: "Send requests from log file",
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
Try to not use global varialbe Try to not use global varialbe
|
||||
Long: "Reads the network log file and sends each request to the specified URL",
|
||||
Example: "frostfs-s3-playback --config <config_path> run [--endpoint=<endpoint>] [--log=<log_path>]",
|
||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
It's better to use It's better to use `Println` function from `cmd *cobra.Command`
|
||||
if rootCmd.PersistentPreRunE != nil {
|
||||
if err := rootCmd.PersistentPreRunE(cmd, args); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
_ = viper.BindPFlag(logPathFlag, cmd.Flags().Lookup(logPathFlag))
|
||||
_ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag))
|
||||
|
||||
return nil
|
||||
},
|
||||
RunE: run,
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
Let's move this near to the place where it's used. (e.g. before or after line Let's move this near to the place where it's used. (e.g. before or after line `for sc.Scan() {`)
|
||||
}
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
We can omit
We can omit `var`:
```
creds := request.Credentials{/*...*/}
```
|
||||
|
||||
pogpp
commented
credentials.access_key & credentials.secret_key credentials.access_key & credentials.secret_key
should be constants
|
||||
func init() {
|
||||
runCmd.Flags().String(logPathFlag, "./request.log", "log file path")
|
||||
runCmd.Flags().String(endpointFlag, "", "endpoint URL")
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
We should use context from command.
We should use context from command.
```
func run (cmd *cobra.Command, _ []string) error {
// ...
ctx := context.WithValue(cmd.Context(), ...)
// ...
}
```
|
||||
}
|
||||
|
||||
func logResponse(cmd *cobra.Command, id int, resp *http.Response, logReq request.LoggedRequest) {
|
||||
cmd.Println(strconv.Itoa(id)+")", logReq.Method, logReq.URI)
|
||||
cmd.Println(resp.Status)
|
||||
if resp.ContentLength != 0 {
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, resp.ContentLength))
|
||||
if err != nil {
|
||||
cmd.Println(id, err)
|
||||
}
|
||||
cmd.Println(string(body))
|
||||
}
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
I think this isn't necessary. We can create new map in I think this isn't necessary. We can create new map in `SetMultipartUpload` if it doesn't exist yet
nzinkevich
commented
But then SetMultipartUpload should also return an updated context or I should pass a *cobra.Command argument to it for SetContext(ctx). I think both ways look worse. But then SetMultipartUpload should also return an updated context or I should pass a *cobra.Command argument to it for SetContext(ctx). I think both ways look worse.
|
||||
cmd.Println()
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
Why do we do this? Why do we do this?
It seems we can pass only context to `playback` function
|
||||
}
|
||||
|
||||
func run(cmd *cobra.Command, _ []string) error {
|
||||
ctx := request.SetCredentials(
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
It's quite odd to set buffer that has size of full file to read. I believe we can skip setting this param at all (or we can configure this buffer size). It's quite odd to set buffer that has size of full file to read. I believe we can skip setting this param at all (or we can configure this buffer size).
|
||||
cmd.Context(),
|
||||
viper.GetString(awsAccessKey),
|
||||
viper.GetString(awsSecretKey),
|
||||
)
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
Please, use Please, use `cmd.Println`
|
||||
ctx = request.WithMultiparts(ctx)
|
||||
|
||||
file, err := os.Open(viper.GetString(logPathFlag))
|
||||
if err != nil {
|
||||
return err
|
||||
dkirillov
commented
We can use We can use `viper.GetDuration`
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
reader := bufio.NewReader(file)
|
||||
|
||||
client := &http.Client{
|
||||
Transport: http.DefaultTransport,
|
||||
Timeout: viper.GetDuration(httpTimeout),
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
It seems message should be It seems message should be `failed to prepare request:`
|
||||
}
|
||||
|
||||
if viper.GetBool(skipVerifyTLS) {
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
Don't use default client. It has no timeout so in theory it can hang out Don't use default client. It has no timeout so in theory it can hang out
|
||||
client.Transport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
|
||||
}
|
||||
|
||||
id := 1
|
||||
for {
|
||||
logReq, err := getRequestFromLog(reader)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
cmd.PrintErrln(strconv.Itoa(id)+")", "failed to parse request", err)
|
||||
id++
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("interrupted: %w", ctx.Err())
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
Use:
Use:
```
r, err := request.NewRequest(viper.GetString(endpointFlag), data)
```
|
||||
default:
|
||||
resp, err := playback(ctx, logReq, client)
|
||||
if err != nil {
|
||||
cmd.PrintErrln(strconv.Itoa(id)+")", "failed to playback request:", err)
|
||||
id++
|
||||
continue
|
||||
}
|
||||
logResponse(cmd, id, resp, logReq)
|
||||
dkirillov
commented
Why don't we invoke Why don't we invoke `request.SwapUploadID` and `request.Sign` inside `request.NewRequest`?
nzinkevich
commented
request.NewRequest was removed, so considered to leave these functions in the prepareRequest request.NewRequest was removed, so considered to leave these functions in the prepareRequest
|
||||
id++
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
pogpp
commented
blank line blank line
|
||||
}
|
||||
|
||||
func getRequestFromLog(reader *bufio.Reader) (request.LoggedRequest, error) {
|
||||
var logReq request.LoggedRequest
|
||||
req, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return logReq, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal([]byte(req), &logReq)
|
||||
if err != nil {
|
||||
return logReq, err
|
||||
}
|
||||
|
||||
return logReq, nil
|
||||
}
|
||||
|
||||
// playback creates http.Request from LoggedRequest and sends it to specified endpoint.
|
||||
func playback(ctx context.Context, logReq request.LoggedRequest, client *http.Client) (*http.Response, error) {
|
||||
r, err := prepareRequest(ctx, logReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to prepare request: %w", err)
|
||||
}
|
||||
resp, err := client.Do(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
|
||||
alexvanin marked this conversation as resolved
Outdated
alexvanin
commented
We definitely should use We definitely should use `http.NewRequestWithContext` here. Network request may take some time, we don't want to wait in case of `SIGTERM` or any other context cancellation.
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response body: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if err = request.HandleResponse(ctx, r, respBody, logReq.Response); err != nil {
|
||||
return nil, fmt.Errorf("failed to register multipart upload: %w", err)
|
||||
}
|
||||
resp.Body = io.NopCloser(bytes.NewBuffer(respBody))
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// prepareRequest creates request from logs and modifies its signature and uploadId (if presents).
|
||||
func prepareRequest(ctx context.Context, logReq request.LoggedRequest) (*http.Request, error) {
|
||||
r, err := http.NewRequestWithContext(ctx, logReq.Method, viper.GetString(endpointFlag)+logReq.URI,
|
||||
bytes.NewReader(logReq.Body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.Header = logReq.Header
|
||||
|
||||
err = request.SwapUploadID(ctx, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sha256hash := sha256.New()
|
||||
sha256hash.Write(logReq.Body)
|
||||
r.Header.Set(auth.AmzContentSHA256, hex.EncodeToString(sha256hash.Sum(nil)))
|
||||
if r.Header.Get(api.ContentMD5) != "" {
|
||||
sha256hash.Reset()
|
||||
md5hash := md5.New()
|
||||
md5hash.Write(logReq.Body)
|
||||
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(md5hash.Sum(nil)))
|
||||
}
|
||||
err = request.Sign(ctx, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
74
cmd/s3-playback/request/multipart.go
Normal file
|
@ -0,0 +1,74 @@
|
|||
package request
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type MultipartUpload struct {
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
Try to not use global varialbe Try to not use global varialbe
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
|
||||
Bucket string `json:"bucket" xml:"Bucket"`
|
||||
Key string `json:"key" xml:"Key"`
|
||||
UploadID string `json:"uploadId" xml:"UploadId"`
|
||||
}
|
||||
|
||||
func WithMultiparts(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, multipartKey{}, map[string]MultipartUpload{})
|
||||
}
|
||||
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
It's better to rename this function to something like It's better to rename this function to something like `HandleResponse`
|
||||
func SetMultipartUpload(ctx context.Context, oldUploadID string, upload MultipartUpload) error {
|
||||
mparts, ok := ctx.Value(multipartKey{}).(map[string]MultipartUpload)
|
||||
if !ok {
|
||||
return errors.New("multiparts not set")
|
||||
}
|
||||
mparts[oldUploadID] = upload
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetMultipart(ctx context.Context, oldUploadID string) (MultipartUpload, error) {
|
||||
mparts, ok := ctx.Value(multipartKey{}).(map[string]MultipartUpload)
|
||||
if !ok {
|
||||
return MultipartUpload{}, errors.New("no multipart map set")
|
||||
}
|
||||
|
||||
return mparts[oldUploadID], nil
|
||||
}
|
||||
|
||||
func HandleResponse(ctx context.Context, r *http.Request, resp []byte, logResponse []byte) error {
|
||||
var mpart, mpartOld MultipartUpload
|
||||
if r.Method == "POST" && r.URL.Query().Has("uploads") {
|
||||
err1 := xml.Unmarshal(resp, &mpart)
|
||||
err2 := xml.Unmarshal(logResponse, &mpartOld)
|
||||
if err1 != nil || err2 != nil {
|
||||
return errors.New("xml unmarshal error")
|
||||
}
|
||||
if mpartOld.UploadID != "" {
|
||||
if err := SetMultipartUpload(ctx, mpartOld.UploadID, mpart); err != nil {
|
||||
return err
|
||||
}
|
||||
pogpp
commented
blank line blank line
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func SwapUploadID(ctx context.Context, r *http.Request) error {
|
||||
var uploadID string
|
||||
query := r.URL.Query()
|
||||
|
||||
if query.Has("uploadId") {
|
||||
uploadID = query.Get("uploadId")
|
||||
mpart, err := GetMultipart(ctx, uploadID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get multipart upload: %w", err)
|
||||
}
|
||||
query.Set("uploadId", mpart.UploadID)
|
||||
r.URL.RawQuery = query.Encode()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
101
cmd/s3-playback/request/request.go
Normal file
|
@ -0,0 +1,101 @@
|
|||
package request
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
pogpp
commented
we have own signer v4 we have own [signer](https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/src/branch/master/api/auth/signer/v4/v4.go#L167) v4
nzinkevich
commented
I chose AWS SDK v2 because it has the Sign HTTP method, which is convenient for updating the S3 request signature. SDK v2 is already set in the go.mod file, but I also added the "github.com/aws/aws-sdk-go-v2/aws/credentials" package. @dkirillov I chose AWS SDK v2 because it has the Sign HTTP method, which is convenient for updating the S3 request signature. SDK v2 is already set in the go.mod file, but I also added the "github.com/aws/aws-sdk-go-v2/aws/credentials" package. @dkirillov
|
||||
)
|
||||
|
||||
var (
|
||||
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
|
||||
authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
|
||||
pogpp
commented
no usages no usages
alexvanin marked this conversation as resolved
Outdated
alexvanin
commented
We need some unit-tests to validate this regexp, just in case. We need some unit-tests to validate this regexp, just in case.
dkirillov
commented
Maybe we can also reuse this regexp from Maybe we can also reuse this regexp from `api/auth/center.go`
|
||||
ErrNoMatches = errors.New("no matches found")
|
||||
)
|
||||
|
||||
type (
|
||||
LoggedRequest struct {
|
||||
From string `json:"from"`
|
||||
URI string `json:"URI"`
|
||||
Method string `json:"method"`
|
||||
Query url.Values `json:"query"`
|
||||
Body []byte `json:"body"`
|
||||
Header http.Header `json:"headers"`
|
||||
Response []byte `json:"response"`
|
||||
}
|
||||
Credentials struct {
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
dkirillov marked this conversation as resolved
Outdated
dkirillov
commented
Let's write something like
Let's write something like
```
type credKeyType struct{}
func SetCredentials(ctx context.Context, creds Credentials) context.Context {
return context.WithValue(ctx, credKeyType{}, creds)
}
func GetCredentials(ctx context.Context) (Credentials, error) {
creds, ok := ctx.Value(credKeyType{}).(Credentials)
if !ok {
return Credentials{}, errors.New("credentials is missing in contex")
}
return creds, nil
}
```
|
||||
}
|
||||
contextKey struct{}
|
||||
multipartKey struct{}
|
||||
)
|
||||
|
||||
func SetCredentials(ctx context.Context, accessKey, secretKey string) context.Context {
|
||||
return context.WithValue(ctx, contextKey{},
|
||||
Credentials{
|
||||
AccessKey: accessKey, SecretKey: secretKey,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func GetCredentials(ctx context.Context) (Credentials, error) {
|
||||
val, ok := ctx.Value(contextKey{}).(Credentials)
|
||||
if !ok {
|
||||
return val, errors.New("credentials not set")
|
||||
}
|
||||
|
||||
return val, nil
|
||||
}
|
||||
|
||||
// Sign replace Authorization header with new Access key id and Signature values.
|
||||
func Sign(ctx context.Context, r *http.Request) error {
|
||||
creds, err := GetCredentials(ctx)
|
||||
if err != nil {
|
||||
return errors.New("failed to get credentials")
|
||||
}
|
||||
credProvider, err := credentials.NewStaticCredentialsProvider(creds.AccessKey,
|
||||
creds.SecretKey, "").Retrieve(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
authInfo, err := parseAuthHeader(r.Header.Get(auth.AuthorizationHdr))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.Header[auth.AuthorizationHdr][0] = strings.Replace(r.Header[auth.AuthorizationHdr][0],
|
||||
authInfo["access_key_id"], creds.AccessKey, 1)
|
||||
|
||||
signer := v4.NewSigner()
|
||||
signatureDateTimeStr := r.Header.Get(api.AmzDate)
|
||||
signatureDateTime, err := time.Parse("20060102T150405Z", signatureDateTimeStr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = signer.SignHTTP(ctx, credProvider, r, r.Header.Get(api.AmzContentSha256),
|
||||
authInfo["service"], authInfo["region"], signatureDateTime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
dkirillov marked this conversation as resolved
Outdated
pogpp
commented
do we need this? do we need this?
nzinkevich
commented
I will use this later to replace the 'X-Amz-Content-SHA256' header. This is useful to ensure that the hash relates to the actual body of the request, which can sometimes be incomplete when saved in logs. I will use this later to replace the 'X-Amz-Content-SHA256' header. This is useful to ensure that the hash relates to the actual body of the request, which can sometimes be incomplete when saved in logs.
dkirillov
commented
Then we should add this function when we will actually use it rather than keep commented code. Then we should add this function when we will actually use it rather than keep commented code.
|
||||
func parseAuthHeader(authHeader string) (map[string]string, error) {
|
||||
authInfo := auth.NewRegexpMatcher(authorizationFieldRegexp).GetSubmatches(authHeader)
|
||||
if len(authInfo) == 0 {
|
||||
return nil, ErrNoMatches
|
||||
}
|
||||
|
||||
return authInfo, nil
|
||||
}
|
95
cmd/s3-playback/request/request_test.go
Normal file
|
@ -0,0 +1,95 @@
|
|||
package request
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func withoutValue(data map[string]string, field string) map[string]string {
|
||||
result := make(map[string]string)
|
||||
for k, v := range data {
|
||||
result[k] = v
|
||||
}
|
||||
result[field] = ""
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func TestParseAuthHeader(t *testing.T) {
|
||||
defaultHeader := "AWS4-HMAC-SHA256 Credential=oid0cid/20210809/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2811ccb9e242f41426738fb1f"
|
||||
|
||||
defaultAuthInfo := map[string]string{
|
||||
"access_key_id": "oid0cid",
|
||||
"service": "s3",
|
||||
"region": "us-east-1",
|
||||
"v4_signature": "2811ccb9e242f41426738fb1f",
|
||||
"signed_header_fields": "host;x-amz-content-sha256;x-amz-date",
|
||||
"date": "20210809",
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
title string
|
||||
header string
|
||||
err error
|
||||
expected map[string]string
|
||||
}{
|
||||
{
|
||||
title: "correct full header",
|
||||
header: defaultHeader,
|
||||
err: nil,
|
||||
expected: defaultAuthInfo,
|
||||
},
|
||||
{
|
||||
title: "correct with empty region",
|
||||
header: strings.Replace(defaultHeader, "/us-east-1/", "//", 1),
|
||||
err: nil,
|
||||
expected: withoutValue(defaultAuthInfo, "region"),
|
||||
},
|
||||
{
|
||||
title: "empty access key",
|
||||
header: strings.Replace(defaultHeader, "oid0cid", "", 1),
|
||||
err: ErrNoMatches,
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
title: "empty service",
|
||||
header: strings.Replace(defaultHeader, "/s3/", "//", 1),
|
||||
err: ErrNoMatches,
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
title: "empty date",
|
||||
header: strings.Replace(defaultHeader, "/20210809/", "//", 1),
|
||||
err: ErrNoMatches,
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
title: "empty v4_signature",
|
||||
header: strings.Replace(defaultHeader, "Signature=2811ccb9e242f41426738fb1f",
|
||||
"Signature=", 1),
|
||||
err: ErrNoMatches,
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
title: "empty signed_fields",
|
||||
header: strings.Replace(defaultHeader, "SignedHeaders=host;x-amz-content-sha256;x-amz-date",
|
||||
"SignedHeaders=", 1),
|
||||
err: ErrNoMatches,
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
title: "empty signed_fields",
|
||||
header: strings.Replace(defaultHeader, "SignedHeaders=host;x-amz-content-sha256;x-amz-date",
|
||||
"SignedHeaders=", 1),
|
||||
err: ErrNoMatches,
|
||||
expected: nil,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.title, func(t *testing.T) {
|
||||
authInfo, err := parseAuthHeader(tc.header)
|
||||
require.ErrorIs(t, err, tc.err, tc.header)
|
||||
require.Equal(t, tc.expected, authInfo, tc.header)
|
||||
})
|
||||
}
|
||||
}
|
|
@ -377,6 +377,11 @@ logger:
|
|||
|
||||
### `http_logging` section
|
||||
|
||||
Could be enabled only in builds with `loghttp` build tag. To build with `loghttp` tag, pass `GOFLAGS` var to `make`:
|
||||
```bash
|
||||
make GOFLAGS="-tags=loghttp" [target]
|
||||
alexvanin
commented
Needs to be updated with new make command. Needs to be updated with new make command.
|
||||
```
|
||||
|
||||
```yaml
|
||||
http_logging:
|
||||
enabled: false
|
||||
|
|
47
docs/playback.md
Normal file
|
@ -0,0 +1,47 @@
|
|||
# FrostFS S3 Playback
|
||||
|
||||
Playback is a tool to reproduce queries to `frostfs-s3-gw` in dev environment. Network logs could be
|
||||
gathered from `s3-gw` via HTTP Logger which could be enabled on build with `loghttp` build tag
|
||||
and `http_logging.enabled` option set to `true` in `s3-gw` configuration.
|
||||
|
||||
## Commands
|
||||
|
||||
`run` - reads log file and reproduces send requests from it to specified endpoint
|
||||
|
||||
#### Example
|
||||
```bash
|
||||
frostfs-s3-playback --config <config_path> run [--endpoint=<endpoint>] [--log=<log_path>]
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Playback accepts configuration file path in yaml with corresponding options:
|
||||
```yaml
|
||||
endpoint: http://localhost:8084
|
||||
log: ./request.log
|
||||
env: .env
|
||||
credentials:
|
||||
access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD
|
||||
secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30
|
||||
http_timeout: 60s
|
||||
skip_verify_tls: true
|
||||
```
|
||||
Configuration path is passed via required `--config` flag.
|
||||
If corresponding flag is set, it overrides parameter from config.
|
||||
|
||||
### Configuration parameters
|
||||
|
||||
#### Global params
|
||||
| # | Config parameter name | Flag name | Type | Default value | Description |
|
||||
|---|-----------------------|-----------------|----------|---------------|--------------------------------------------------------|
|
||||
| 1 | - | config | string | - | config file path (e.g. `./config/playback.yaml`) |
|
||||
| 2 | http_timeout | http-timeout | duration | 60s | http request timeout |
|
||||
| 3 | skip_verify_tls | skip-verify-tls | bool | false | skips tls certificate verification for https endpoints |
|
||||
| 4 | credentials.accessKey | - | string | - | AWS access key id |
|
||||
| 5 | credentials.secretKey | - | string | - | AWS secret key |
|
||||
|
||||
#### `run` command parameters
|
||||
| # | Config parameter name | Flag name | Type | Default value | Description |
|
||||
|---|-----------------------|-----------|--------|---------------|--------------------------------------------------------|
|
||||
| 1 | endpoint | endpoint | string | - | s3-gw endpoint URL |
|
||||
| 2 | log | log | string | ./request.log | path to log file, could be either absolute or relative |
|
5
go.mod
|
@ -10,7 +10,8 @@ require (
|
|||
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a
|
||||
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
|
||||
github.com/aws/aws-sdk-go v1.44.6
|
||||
github.com/aws/aws-sdk-go-v2 v1.18.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.30.3
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.27
|
||||
github.com/bluele/gcache v0.0.2
|
||||
github.com/go-chi/chi/v5 v5.0.8
|
||||
github.com/google/uuid v1.3.1
|
||||
|
@ -43,7 +44,7 @@ require (
|
|||
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
|
||||
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
|
||||
github.com/aws/smithy-go v1.13.5 // indirect
|
||||
github.com/aws/smithy-go v1.20.3 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
|
|
11
go.sum
|
@ -64,10 +64,12 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8
|
|||
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
|
||||
github.com/aws/aws-sdk-go v1.44.6 h1:Y+uHxmZfhRTLX2X3khkdxCoTZAyGEX21aOUHe1U6geg=
|
||||
github.com/aws/aws-sdk-go v1.44.6/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo=
|
||||
github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
|
||||
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
|
||||
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
|
||||
github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
|
||||
github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
|
||||
|
@ -172,7 +174,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
|||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
|
|
blank line