[#369] Request reproducer

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>

cmd/s3-playback/config/playback.yaml

@@ -0,0 +1,7 @@
+endpoint: http://localhost:8084
+log: ./log/request.log
+credentials:
+  access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD
+  secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30
+http_timeout: 60
+skip_verify_tls: false

cmd/s3-playback/main.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/modules"
+)
+
+func main() {
+	ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
+
+	if cmd, err := modules.Execute(ctx); err != nil {
+		cmd.PrintErrln("Error:", err.Error())
+		cmd.PrintErrf("Run '%v --help' for usage.\n", cmd.CommandPath())
+		os.Exit(1)
+	}
+}

cmd/s3-playback/modules/root.go

@@ -0,0 +1,61 @@
+package modules
+
+import (
+	"context"
+	"strings"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+const (
+	configFlag      = "config"
+	httpTimeoutFlag = "http-timeout"
+	skipVerifyTLS   = "skip-verify-tls"
+)
+
+var rootCmd = &cobra.Command{
+	Use:           "frostfs-s3-playback",
+	Version:       version.Version,
+	Short:         "FrostFS S3 Traffic Playback",
+	Long:          "Helps to reproduce s3 commands from log files",
+	Example:       "frostfs-s3-playback --version",
+	SilenceUsage:  true,
+	SilenceErrors: true,
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		return cmd.Help()
+	},
+}
+
+func Execute(ctx context.Context) (*cobra.Command, error) {
+	ctx = ctx
+	return rootCmd.ExecuteContextC(ctx)
+}
+
+func initConfig() {
+	viper.SetConfigFile(viper.GetString(configFlag))
+	viper.ReadInConfig()
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringP(configFlag, "c", "",
+		"configuration filepath")
+	_ = rootCmd.MarkPersistentFlagRequired(configFlag)
+	_ = rootCmd.MarkPersistentFlagFilename(configFlag)
+	viper.BindPFlag(strings.Replace(configFlag, "-", "_", -1),
+		rootCmd.PersistentFlags().Lookup(configFlag))
+
+	rootCmd.PersistentFlags().String(httpTimeoutFlag, "60",
+		"http request timeout")
+	viper.BindPFlag(strings.Replace(httpTimeoutFlag, "-", "_", -1),
+		rootCmd.PersistentFlags().Lookup(httpTimeoutFlag))
+
+	rootCmd.PersistentFlags().Bool(skipVerifyTLS, false,
+		"skip TLS certificate verification")
+	viper.BindPFlag(strings.Replace(skipVerifyTLS, "_", "-", -1),
+		rootCmd.PersistentFlags().Lookup(skipVerifyTLS))
+
+	cobra.OnInitialize(initConfig)
+	rootCmd.AddCommand(runCmd)
+}

cmd/s3-playback/modules/run.go

@@ -0,0 +1,175 @@
+package modules
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/md5"
+	"crypto/sha256"
+	"crypto/tls"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
+	request2 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/request"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+const (
+	logPathFlag  = "log"
+	endpointFlag = "endpoint"
+)
+
+var runCmd = &cobra.Command{
+	Use:     "run",
+	Short:   "Send requests from log file",
+	Long:    "Reads the network log file and sends each request to the specified URL",
+	Example: "frostfs-s3-playback --config <config_path> run [--endpoint=<endpoint>] [--log=<log_path>]",
+	RunE:    run,
+}
+
+func init() {
+	runCmd.Flags().String(logPathFlag, "", "Log file path")
+	viper.BindPFlag(logPathFlag, runCmd.Flags().Lookup(logPathFlag))
+	runCmd.Flags().String(endpointFlag, "", "Endpoint URL")
+	viper.BindPFlag(endpointFlag, runCmd.Flags().Lookup(endpointFlag))
+}
+
+func logResponse(cmd *cobra.Command, id int, resp *http.Response, logReq request2.LoggedRequest) {
+	cmd.Println(strconv.Itoa(id)+")", logReq.Method, logReq.URI)
+	cmd.Println(resp.Status)
+	if resp.ContentLength != 0 {
+		body, err := io.ReadAll(io.LimitReader(resp.Body, resp.ContentLength))
+		if err != nil {
+			cmd.Println(id, err)
+		}
+		cmd.Println(string(body))
+	}
+	cmd.Println()
+}
+
+func run(cmd *cobra.Command, _ []string) error {
+	ctx := request2.SetCredentials(
+		cmd.Context(),
+		viper.GetString("credentials.access_key"),
+		viper.GetString("credentials.secret_key"),
+	)
+	ctx = request2.WithMultiparts(ctx)
+
+	cmd.SetContext(ctx)
+
+	file, err := os.Open(viper.GetString(logPathFlag))
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	reader := bufio.NewReader(file)
+
+	client := &http.Client{
+		Transport: http.DefaultTransport,
+		Timeout:   time.Second * time.Duration(viper.GetInt64(httpTimeoutFlag)),
+	}
+
+	if viper.GetBool(skipVerifyTLS) {
+		client.Transport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+	id := 1
+	for {
+		req, err := reader.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				break
+			} else {
+				return err
+			}
+		}
+
+		var logReq request2.LoggedRequest
+		if err = json.Unmarshal([]byte(req), &logReq); err != nil {
+			cmd.PrintErrln(strconv.Itoa(id)+")", "failed to parse request", err)
+			id++
+			continue
+		}
+
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("interrupted: %w", ctx.Err())
+		default:
+			resp, err := playback(cmd, logReq, client)
+			if err != nil {
+				cmd.PrintErrln(strconv.Itoa(id)+")", "failed to playback request:", err)
+				id++
+				continue
+			}
+			logResponse(cmd, id, resp, logReq)
+			id++
+		}
+	}
+
+	return nil
+}
+
+// playback creates http.Request from LoggedRequest and sends it to specified endpoint.
+func playback(cmd *cobra.Command, logReq request2.LoggedRequest, client *http.Client) (*http.Response, error) {
+	r, err := prepareRequest(cmd, logReq)
+	if err != nil {
+		return nil, fmt.Errorf("failed to prepare request: %w", err)
+	}
+	resp, err := client.Do(r)
+	if err != nil {
+		return nil, fmt.Errorf("failed to send request: %w", err)
+	}
+
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if err = request2.HandleResponse(cmd.Context(), r, respBody, logReq.Response); err != nil {
+		return nil, fmt.Errorf("failed to register multipart upload: %w", err)
+	}
+	resp.Body = io.NopCloser(bytes.NewBuffer(respBody))
+
+	return resp, nil
+}
+
+// prepareRequest creates request from logs and modifies its signature and uploadId (if presents).
+func prepareRequest(cmd *cobra.Command, logReq request2.LoggedRequest) (*http.Request, error) {
+	r, err := http.NewRequest(logReq.Method, viper.GetString(endpointFlag)+logReq.URI,
+		bytes.NewReader(logReq.Body))
+	if err != nil {
+		return nil, err
+	}
+	r.Header = logReq.Header
+
+	err = request2.SwapUploadID(cmd.Context(), r)
+	if err != nil {
+		return nil, err
+	}
+
+	sha256hash := sha256.New()
+	sha256hash.Write(logReq.Body)
+	r.Header.Set(auth.AmzContentSHA256, hex.EncodeToString(sha256hash.Sum(nil)))
+	if r.Header.Get(api.ContentMD5) != "" {
+		sha256hash.Reset()
+		md5hash := md5.New()
+		md5hash.Write(logReq.Body)
+		r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(md5hash.Sum(nil)))
+	}
+	err = request2.Sign(cmd, r)
+	if err != nil {
+		return nil, err
+	}
+	return r, nil
+}

cmd/s3-playback/request/multipart.go

@@ -0,0 +1,74 @@
+package request
+
+import (
+	"context"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"net/http"
+)
+
+type MultipartUpload struct {
+	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
+	Bucket   string   `json:"bucket" xml:"Bucket"`
+	Key      string   `json:"key" xml:"Key"`
+	UploadID string   `json:"uploadId" xml:"UploadId"`
+}
+
+func WithMultiparts(ctx context.Context) context.Context {
+	return context.WithValue(ctx, multipartKey{}, map[string]MultipartUpload{})
+}
+
+func SetMultipartUpload(ctx context.Context, oldUploadId string, upload MultipartUpload) error {
+	mparts, ok := ctx.Value(multipartKey{}).(map[string]MultipartUpload)
+	if !ok {
+		return errors.New("multiparts not set")
+	}
+	mparts[oldUploadId] = upload
+
+	return nil
+}
+
+func GetMultipart(ctx context.Context, oldUploadId string) (MultipartUpload, error) {
+	mparts, ok := ctx.Value(multipartKey{}).(map[string]MultipartUpload)
+	if !ok {
+		return MultipartUpload{}, errors.New("no multipart map set")
+	}
+
+	return mparts[oldUploadId], nil
+}
+
+func HandleResponse(ctx context.Context, r *http.Request, resp []byte, logResponse []byte) error {
+	var mpart, mpartOld MultipartUpload
+	if r.Method == "POST" && r.URL.Query().Has("uploads") {
+		err1 := xml.Unmarshal(resp, &mpart)
+		err2 := xml.Unmarshal(logResponse, &mpartOld)
+		if err1 != nil || err2 != nil {
+			return errors.New("xml unmarshal error")
+		}
+		if mpartOld.UploadID != "" {
+			if err := SetMultipartUpload(ctx, mpartOld.UploadID, mpart); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func SwapUploadID(ctx context.Context, r *http.Request) error {
+	var uploadID string
+	query := r.URL.Query()
+
+	if query.Has("uploadId") {
+		uploadID = query.Get("uploadId")
+		mpart, err := GetMultipart(ctx, uploadID)
+		if err != nil {
+			return fmt.Errorf("failed to get multipart upload: %w", err)
+		}
+		query.Set("uploadId", mpart.UploadID)
+		r.URL.RawQuery = query.Encode()
+	}
+
+	return nil
+}

cmd/s3-playback/request/request.go

@@ -0,0 +1,101 @@
+package request
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/spf13/cobra"
+)
+
+// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
+var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
+
+type (
+	LoggedRequest struct {
+		From     string      `json:"from"`
+		URI      string      `json:"URI"`
+		Method   string      `json:"method"`
+		Query    url.Values  `json:"query"`
+		Body     []byte      `json:"body"`
+		Header   http.Header `json:"headers"`
+		Response []byte      `json:"response"`
+	}
+	Credentials struct {
+		AccessKey string
+		SecretKey string
+	}
+	contextKey   struct{}
+	multipartKey struct{}
+)
+
+func SetCredentials(ctx context.Context, accessKey, secretKey string) context.Context {
+	return context.WithValue(ctx, contextKey{},
+		Credentials{
+			AccessKey: accessKey, SecretKey: secretKey,
+		},
+	)
+}
+
+func GetCredentials(ctx context.Context) (Credentials, error) {
+	val, ok := ctx.Value(contextKey{}).(Credentials)
+	if !ok {
+		return val, errors.New("credentials not set")
+	}
+
+	return val, nil
+}
+
+// Sign replace Authorization header with new Access key id and Signature values.
+func Sign(cmd *cobra.Command, r *http.Request) error {
+	ctx := cmd.Context()
+	creds, err := GetCredentials(ctx)
+	if err != nil {
+		return errors.New("failed to get credentials")
+	}
+	credProvider, err := credentials.NewStaticCredentialsProvider(creds.AccessKey,
+		creds.SecretKey, "").Retrieve(ctx)
+	if err != nil {
+		return err
+	}
+
+	authInfo, err := parseAuthHeader(r)
+	if err != nil {
+		return err
+	}
+	r.Header[auth.AuthorizationHdr][0] = strings.Replace(r.Header[auth.AuthorizationHdr][0],
+		authInfo["access_key_id"], creds.AccessKey, 1)
+
+	signer := v4.NewSigner()
+	signatureDateTimeStr := r.Header.Get(api.AmzDate)
+	signatureDateTime, err := time.Parse("20060102T150405Z", signatureDateTimeStr)
+	if err != nil {
+		return err
+	}
+
+	err = signer.SignHTTP(ctx, credProvider, r, r.Header.Get(api.AmzContentSha256),
+		authInfo["service"], authInfo["region"], signatureDateTime)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func parseAuthHeader(r *http.Request) (map[string]string, error) {
+	matcher := auth.NewRegexpMatcher(authorizationFieldRegexp)
+	authHeader := r.Header[auth.AuthorizationHdr]
+	if len(authHeader) != 1 {
+		return nil, errors.New("invalid authorization header")
+	}
+	authinfo := matcher.GetSubmatches(authHeader[0])
+	return authinfo, nil
+}

docs/playback.md

@@ -0,0 +1,41 @@
+# FrostFS S3 PlayBack
+
+Playback is a tool to reproduce PoC network activity in dev environment. Network logs could be
+gathered from s3-gw via HTTP Logger which could be enabled on build with `loghttp` build tag 
+and `http_logging` option enabled in app settings.
+
+## Configuration
+
+Playback accepts configuration file path in yaml with corresponding options:
+```yaml
+endpoint: http://localhost:8084
+log: ./request.log
+env: .env
+credentials:
+  access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD
+  secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30
+http_timeout: 360
+skip_verify_tls: true
+```
+Configuration path is passed via required `--config` flag.
+
+### Configuration parameters
+
+#### Persistent params
+| # | Parameter             | Flag name       | Type   | Default value | Description                                                        |
+|---|-----------------------|-----------------|--------|---------------|--------------------------------------------------------------------|
+| 1 | http_timeout          | http-timeout    | int    | 60            | http request timeout                                               |
+| 2 | skip_verify_tls       | skip-verify-tls | bool   | false         | skips tls certificate verification for self-signed https endpoints |
+| 3 | credentials.accessKey | -               | string | -             | aws access key id                                                  |
+| 4 | credentials.secretKey | -               | string | -             | aws secret key                                                     |
+
+#### `run` command parameters
+| # | Parameter | Flag name | Type   | Default value         | Description                                            |
+|---|-----------|-----------|--------|-----------------------|--------------------------------------------------------|
+| 1 | endpoint  | endpoint  | string | http://localhost:8080 | s3-gw endpoint URL                                     |
+| 2 | log       | log       | string | ./request.log         | path to log file, could be either absolute or relative |
+|   |           |           |        |                       |                                                        |
+|   |           |           |        |                       |                                                        |
+|   |           |           |        |                       |                                                        |
+|   |           |           |        |                       |                                                        |
+|   |           |           |        |                       |                                                        |

go.mod

@@ -10,7 +10,8 @@ require (
 	git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a
 	git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
 	github.com/aws/aws-sdk-go v1.44.6
-	github.com/aws/aws-sdk-go-v2 v1.18.1
+	github.com/aws/aws-sdk-go-v2 v1.30.3
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.27
 	github.com/bluele/gcache v0.0.2
 	github.com/go-chi/chi/v5 v5.0.8
 	github.com/google/uuid v1.3.1
@@ -43,7 +44,7 @@ require (
 	git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
 	git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
-	github.com/aws/smithy-go v1.13.5 // indirect
+	github.com/aws/smithy-go v1.20.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect

go.sum

@@ -64,10 +64,12 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8
 github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/aws/aws-sdk-go v1.44.6 h1:Y+uHxmZfhRTLX2X3khkdxCoTZAyGEX21aOUHe1U6geg=
 github.com/aws/aws-sdk-go v1.44.6/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
-github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo=
-github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
-github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
-github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
+github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
+github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
+github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
@@ -172,7 +174,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=