[#369] Request reproducer
All checks were successful
/ DCO (pull_request) Successful in 56s
/ Vulncheck (pull_request) Successful in 1m13s
/ Builds (1.20) (pull_request) Successful in 1m25s
/ Builds (1.21) (pull_request) Successful in 1m28s
/ Lint (pull_request) Successful in 2m40s
/ Tests (1.20) (pull_request) Successful in 1m33s
/ Tests (1.21) (pull_request) Successful in 1m38s

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
Nikita Zinkevich 2024-07-24 14:23:17 +03:00
parent 5d55934748
commit 924f3c0b75
9 changed files with 487 additions and 7 deletions

View file

@ -0,0 +1,7 @@
endpoint: http://localhost:8084
log: ./log/request.log
credentials:
access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD
secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30
http_timeout: 60
skip_verify_tls: false

20
cmd/s3-playback/main.go Normal file
View file

@ -0,0 +1,20 @@
package main
import (
"context"
"os"
"os/signal"
"syscall"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/modules"
)
func main() {
ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
if cmd, err := modules.Execute(ctx); err != nil {
cmd.PrintErrln("Error:", err.Error())
cmd.PrintErrf("Run '%v --help' for usage.\n", cmd.CommandPath())
os.Exit(1)
}
}

View file

@ -0,0 +1,60 @@
package modules
import (
"context"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
const (
configFlag = "config"
httpTimeoutFlag = "http-timeout"
skipVerifyTLS = "skip-verify-tls"
)
var rootCmd = &cobra.Command{
Use: "frostfs-s3-playback",
Version: version.Version,
Short: "FrostFS S3 Traffic Playback",
Long: "Helps to reproduce s3 commands from log files",
Example: "frostfs-s3-playback --version",
SilenceUsage: true,
SilenceErrors: true,
RunE: func(cmd *cobra.Command, _ []string) error {
return cmd.Help()
},
}
func Execute(ctx context.Context) (*cobra.Command, error) {
return rootCmd.ExecuteContextC(ctx)
}
func initConfig() {
viper.SetConfigFile(viper.GetString(configFlag))
_ = viper.ReadInConfig()
}
func init() {
rootCmd.PersistentFlags().StringP(configFlag, "c", "",
"configuration filepath")
_ = rootCmd.MarkPersistentFlagRequired(configFlag)
_ = rootCmd.MarkPersistentFlagFilename(configFlag)
_ = viper.BindPFlag(strings.Replace(configFlag, "-", "_", -1),
rootCmd.PersistentFlags().Lookup(configFlag))
rootCmd.PersistentFlags().String(httpTimeoutFlag, "60",
"http request timeout")
_ = viper.BindPFlag(strings.Replace(httpTimeoutFlag, "-", "_", -1),
rootCmd.PersistentFlags().Lookup(httpTimeoutFlag))
rootCmd.PersistentFlags().Bool(skipVerifyTLS, false,
"skip TLS certificate verification")
_ = viper.BindPFlag(strings.Replace(skipVerifyTLS, "_", "-", -1),
rootCmd.PersistentFlags().Lookup(skipVerifyTLS))
cobra.OnInitialize(initConfig)
rootCmd.AddCommand(runCmd)
}

View file

@ -0,0 +1,175 @@
package modules
import (
"bufio"
"bytes"
"crypto/md5"
"crypto/sha256"
"crypto/tls"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
request2 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/request"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
const (
logPathFlag = "log"
endpointFlag = "endpoint"
)
var runCmd = &cobra.Command{
Use: "run",
Short: "Send requests from log file",
Long: "Reads the network log file and sends each request to the specified URL",
Example: "frostfs-s3-playback --config <config_path> run [--endpoint=<endpoint>] [--log=<log_path>]",
RunE: run,
}
func init() {
runCmd.Flags().String(logPathFlag, "", "Log file path")
_ = viper.BindPFlag(logPathFlag, runCmd.Flags().Lookup(logPathFlag))
runCmd.Flags().String(endpointFlag, "", "Endpoint URL")
_ = viper.BindPFlag(endpointFlag, runCmd.Flags().Lookup(endpointFlag))
}
func logResponse(cmd *cobra.Command, id int, resp *http.Response, logReq request2.LoggedRequest) {
cmd.Println(strconv.Itoa(id)+")", logReq.Method, logReq.URI)
cmd.Println(resp.Status)
if resp.ContentLength != 0 {
body, err := io.ReadAll(io.LimitReader(resp.Body, resp.ContentLength))
if err != nil {
cmd.Println(id, err)
}
cmd.Println(string(body))
}
cmd.Println()
}
func run(cmd *cobra.Command, _ []string) error {
ctx := request2.SetCredentials(
cmd.Context(),
viper.GetString("credentials.access_key"),
viper.GetString("credentials.secret_key"),
)
ctx = request2.WithMultiparts(ctx)
cmd.SetContext(ctx)
file, err := os.Open(viper.GetString(logPathFlag))
if err != nil {
return err
}
defer file.Close()
reader := bufio.NewReader(file)
client := &http.Client{
Transport: http.DefaultTransport,
Timeout: time.Second * time.Duration(viper.GetInt64(httpTimeoutFlag)),
}
if viper.GetBool(skipVerifyTLS) {
client.Transport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
id := 1
for {
req, err := reader.ReadString('\n')
if err != nil {
if err == io.EOF {
break
}
return err
}
var logReq request2.LoggedRequest
if err = json.Unmarshal([]byte(req), &logReq); err != nil {
cmd.PrintErrln(strconv.Itoa(id)+")", "failed to parse request", err)
id++
continue
}
select {
case <-ctx.Done():
return fmt.Errorf("interrupted: %w", ctx.Err())
default:
resp, err := playback(cmd, logReq, client)
if err != nil {
cmd.PrintErrln(strconv.Itoa(id)+")", "failed to playback request:", err)
id++
continue
}
logResponse(cmd, id, resp, logReq)
id++
}
}
return nil
}
// playback creates http.Request from LoggedRequest and sends it to specified endpoint.
func playback(cmd *cobra.Command, logReq request2.LoggedRequest, client *http.Client) (*http.Response, error) {
r, err := prepareRequest(cmd, logReq)
if err != nil {
return nil, fmt.Errorf("failed to prepare request: %w", err)
}
resp, err := client.Do(r)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body: %w", err)
}
defer resp.Body.Close()
if err = request2.HandleResponse(cmd.Context(), r, respBody, logReq.Response); err != nil {
return nil, fmt.Errorf("failed to register multipart upload: %w", err)
}
resp.Body = io.NopCloser(bytes.NewBuffer(respBody))
return resp, nil
}
// prepareRequest creates request from logs and modifies its signature and uploadId (if presents).
func prepareRequest(cmd *cobra.Command, logReq request2.LoggedRequest) (*http.Request, error) {
r, err := http.NewRequest(logReq.Method, viper.GetString(endpointFlag)+logReq.URI,
bytes.NewReader(logReq.Body))
if err != nil {
return nil, err
}
r.Header = logReq.Header
err = request2.SwapUploadID(cmd.Context(), r)
if err != nil {
return nil, err
}
sha256hash := sha256.New()
sha256hash.Write(logReq.Body)
r.Header.Set(auth.AmzContentSHA256, hex.EncodeToString(sha256hash.Sum(nil)))
if r.Header.Get(api.ContentMD5) != "" {
sha256hash.Reset()
md5hash := md5.New()
md5hash.Write(logReq.Body)
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(md5hash.Sum(nil)))
}
err = request2.Sign(cmd, r)
if err != nil {
return nil, err
}
return r, nil
}

View file

@ -0,0 +1,74 @@
package request
import (
"context"
"encoding/xml"
"errors"
"fmt"
"net/http"
)
type MultipartUpload struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
Bucket string `json:"bucket" xml:"Bucket"`
Key string `json:"key" xml:"Key"`
UploadID string `json:"uploadId" xml:"UploadId"`
}
func WithMultiparts(ctx context.Context) context.Context {
return context.WithValue(ctx, multipartKey{}, map[string]MultipartUpload{})
}
func SetMultipartUpload(ctx context.Context, oldUploadID string, upload MultipartUpload) error {
mparts, ok := ctx.Value(multipartKey{}).(map[string]MultipartUpload)
if !ok {
return errors.New("multiparts not set")
}
mparts[oldUploadID] = upload
return nil
}
func GetMultipart(ctx context.Context, oldUploadID string) (MultipartUpload, error) {
mparts, ok := ctx.Value(multipartKey{}).(map[string]MultipartUpload)
if !ok {
return MultipartUpload{}, errors.New("no multipart map set")
}
return mparts[oldUploadID], nil
}
func HandleResponse(ctx context.Context, r *http.Request, resp []byte, logResponse []byte) error {
var mpart, mpartOld MultipartUpload
if r.Method == "POST" && r.URL.Query().Has("uploads") {
err1 := xml.Unmarshal(resp, &mpart)
err2 := xml.Unmarshal(logResponse, &mpartOld)
if err1 != nil || err2 != nil {
return errors.New("xml unmarshal error")
}
if mpartOld.UploadID != "" {
if err := SetMultipartUpload(ctx, mpartOld.UploadID, mpart); err != nil {
return err
}
}
}
return nil
}
func SwapUploadID(ctx context.Context, r *http.Request) error {
var uploadID string
query := r.URL.Query()
if query.Has("uploadId") {
uploadID = query.Get("uploadId")
mpart, err := GetMultipart(ctx, uploadID)
if err != nil {
return fmt.Errorf("failed to get multipart upload: %w", err)
}
query.Set("uploadId", mpart.UploadID)
r.URL.RawQuery = query.Encode()
}
return nil
}

View file

@ -0,0 +1,101 @@
package request
import (
"context"
"errors"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/spf13/cobra"
)
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
type (
LoggedRequest struct {
From string `json:"from"`
URI string `json:"URI"`
Method string `json:"method"`
Query url.Values `json:"query"`
Body []byte `json:"body"`
Header http.Header `json:"headers"`
Response []byte `json:"response"`
}
Credentials struct {
AccessKey string
SecretKey string
}
contextKey struct{}
multipartKey struct{}
)
func SetCredentials(ctx context.Context, accessKey, secretKey string) context.Context {
return context.WithValue(ctx, contextKey{},
Credentials{
AccessKey: accessKey, SecretKey: secretKey,
},
)
}
func GetCredentials(ctx context.Context) (Credentials, error) {
val, ok := ctx.Value(contextKey{}).(Credentials)
if !ok {
return val, errors.New("credentials not set")
}
return val, nil
}
// Sign replace Authorization header with new Access key id and Signature values.
func Sign(cmd *cobra.Command, r *http.Request) error {
ctx := cmd.Context()
creds, err := GetCredentials(ctx)
if err != nil {
return errors.New("failed to get credentials")
}
credProvider, err := credentials.NewStaticCredentialsProvider(creds.AccessKey,
creds.SecretKey, "").Retrieve(ctx)
if err != nil {
return err
}
authInfo, err := parseAuthHeader(r)
if err != nil {
return err
}
r.Header[auth.AuthorizationHdr][0] = strings.Replace(r.Header[auth.AuthorizationHdr][0],
authInfo["access_key_id"], creds.AccessKey, 1)
signer := v4.NewSigner()
signatureDateTimeStr := r.Header.Get(api.AmzDate)
signatureDateTime, err := time.Parse("20060102T150405Z", signatureDateTimeStr)
if err != nil {
return err
}
err = signer.SignHTTP(ctx, credProvider, r, r.Header.Get(api.AmzContentSha256),
authInfo["service"], authInfo["region"], signatureDateTime)
if err != nil {
return err
}
return nil
}
func parseAuthHeader(r *http.Request) (map[string]string, error) {
matcher := auth.NewRegexpMatcher(authorizationFieldRegexp)
authHeader := r.Header[auth.AuthorizationHdr]
if len(authHeader) != 1 {
return nil, errors.New("invalid authorization header")
}
authinfo := matcher.GetSubmatches(authHeader[0])
return authinfo, nil
}

41
docs/playback.md Normal file
View file

@ -0,0 +1,41 @@
# FrostFS S3 PlayBack
Playback is a tool to reproduce PoC network activity in dev environment. Network logs could be
gathered from s3-gw via HTTP Logger which could be enabled on build with `loghttp` build tag
and `http_logging` option enabled in app settings.
## Configuration
Playback accepts configuration file path in yaml with corresponding options:
```yaml
endpoint: http://localhost:8084
log: ./request.log
env: .env
credentials:
access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD
secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30
http_timeout: 360
skip_verify_tls: true
```
Configuration path is passed via required `--config` flag.
### Configuration parameters
#### Persistent params
| # | Parameter | Flag name | Type | Default value | Description |
|---|-----------------------|-----------------|--------|---------------|--------------------------------------------------------------------|
| 1 | http_timeout | http-timeout | int | 60 | http request timeout |
| 2 | skip_verify_tls | skip-verify-tls | bool | false | skips tls certificate verification for self-signed https endpoints |
| 3 | credentials.accessKey | - | string | - | aws access key id |
| 4 | credentials.secretKey | - | string | - | aws secret key |
#### `run` command parameters
| # | Parameter | Flag name | Type | Default value | Description |
|---|-----------|-----------|--------|-----------------------|--------------------------------------------------------|
| 1 | endpoint | endpoint | string | http://localhost:8080 | s3-gw endpoint URL |
| 2 | log | log | string | ./request.log | path to log file, could be either absolute or relative |
| | | | | | |
| | | | | | |
| | | | | | |
| | | | | | |
| | | | | | |

5
go.mod
View file

@ -10,7 +10,8 @@ require (
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
github.com/aws/aws-sdk-go v1.44.6
github.com/aws/aws-sdk-go-v2 v1.18.1
github.com/aws/aws-sdk-go-v2 v1.30.3
github.com/aws/aws-sdk-go-v2/credentials v1.17.27
github.com/bluele/gcache v0.0.2
github.com/go-chi/chi/v5 v5.0.8
github.com/google/uuid v1.3.1
@ -43,7 +44,7 @@ require (
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/aws/smithy-go v1.20.3 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect

11
go.sum
View file

@ -64,10 +64,12 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
github.com/aws/aws-sdk-go v1.44.6 h1:Y+uHxmZfhRTLX2X3khkdxCoTZAyGEX21aOUHe1U6geg=
github.com/aws/aws-sdk-go v1.44.6/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo=
github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI=
github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
@ -172,7 +174,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=