From c8d0e7f8554dd4453583fc7f6fec4d19c7239be3 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Wed, 24 Jul 2024 14:23:17 +0300 Subject: [PATCH] [#369] Request reproducer Signed-off-by: Nikita Zinkevich --- cmd/s3-playback/config/playback.yaml | 7 ++ cmd/s3-playback/main.go | 20 +++ cmd/s3-playback/modules/root.go | 59 +++++++++ cmd/s3-playback/modules/run.go | 177 +++++++++++++++++++++++++++ cmd/s3-playback/request/multipart.go | 74 +++++++++++ cmd/s3-playback/request/request.go | 101 +++++++++++++++ docs/playback.md | 41 +++++++ go.mod | 5 +- go.sum | 11 +- 9 files changed, 488 insertions(+), 7 deletions(-) create mode 100644 cmd/s3-playback/config/playback.yaml create mode 100644 cmd/s3-playback/main.go create mode 100644 cmd/s3-playback/modules/root.go create mode 100644 cmd/s3-playback/modules/run.go create mode 100644 cmd/s3-playback/request/multipart.go create mode 100644 cmd/s3-playback/request/request.go create mode 100644 docs/playback.md diff --git a/cmd/s3-playback/config/playback.yaml b/cmd/s3-playback/config/playback.yaml new file mode 100644 index 0000000..fe27223 --- /dev/null +++ b/cmd/s3-playback/config/playback.yaml @@ -0,0 +1,7 @@ +endpoint: http://localhost:8084 +log: ./log/request.log +credentials: + access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD + secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30 +http_timeout: 60 +skip_verify_tls: false diff --git a/cmd/s3-playback/main.go b/cmd/s3-playback/main.go new file mode 100644 index 0000000..fc3f25b --- /dev/null +++ b/cmd/s3-playback/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/modules" +) + +func main() { + ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + + if cmd, err := modules.Execute(ctx); err != nil { + cmd.PrintErrln("Error:", err.Error()) + cmd.PrintErrf("Run '%v --help' for usage.\n", cmd.CommandPath()) + os.Exit(1) + } +} diff --git a/cmd/s3-playback/modules/root.go b/cmd/s3-playback/modules/root.go new file mode 100644 index 0000000..d9431f3 --- /dev/null +++ b/cmd/s3-playback/modules/root.go @@ -0,0 +1,59 @@ +package modules + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + configPath = "config" + configPathFlag = "config" + httpTimeout = "http_timeout" + httpTimeoutFlag = "http-timeout" + skipVerifyTLS = "skip_verify_tls" + skipVerifyTLSFlag = "skip-verify-tls" +) + +var rootCmd = &cobra.Command{ + Use: "frostfs-s3-playback", + Version: version.Version, + Short: "FrostFS S3 Traffic Playback", + Long: "Helps to reproduce s3 commands from log files", + Example: "frostfs-s3-playback --version", + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, _ []string) error { + return cmd.Help() + }, +} + +func Execute(ctx context.Context) (*cobra.Command, error) { + return rootCmd.ExecuteContextC(ctx) +} + +func initConfig() { + viper.SetConfigFile(viper.GetString(configPath)) + _ = viper.ReadInConfig() +} + +func init() { + rootCmd.PersistentFlags().StringP(configPathFlag, "c", "", + "configuration filepath") + _ = rootCmd.MarkPersistentFlagRequired(configPathFlag) + _ = rootCmd.MarkPersistentFlagFilename(configPathFlag) + _ = viper.BindPFlag(configPath, rootCmd.PersistentFlags().Lookup(configPathFlag)) + + rootCmd.PersistentFlags().String(httpTimeoutFlag, "60", + "http request timeout") + _ = viper.BindPFlag(httpTimeout, rootCmd.PersistentFlags().Lookup(httpTimeoutFlag)) + + rootCmd.PersistentFlags().Bool(skipVerifyTLSFlag, false, + "skip TLS certificate verification") + _ = viper.BindPFlag(skipVerifyTLS, rootCmd.PersistentFlags().Lookup(skipVerifyTLSFlag)) + + cobra.OnInitialize(initConfig) + rootCmd.AddCommand(runCmd) +} diff --git a/cmd/s3-playback/modules/run.go b/cmd/s3-playback/modules/run.go new file mode 100644 index 0000000..f9489f9 --- /dev/null +++ b/cmd/s3-playback/modules/run.go @@ -0,0 +1,177 @@ +package modules + +import ( + "bufio" + "bytes" + "crypto/md5" + "crypto/sha256" + "crypto/tls" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth" + request2 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/request" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + logPathFlag = "log" + endpointFlag = "endpoint" + awsAccessKey = "credentials.access_key" + awsSecretKey = "credentials.secret_key" +) + +var runCmd = &cobra.Command{ + Use: "run", + Short: "Send requests from log file", + Long: "Reads the network log file and sends each request to the specified URL", + Example: "frostfs-s3-playback --config run [--endpoint=] [--log=]", + RunE: run, +} + +func init() { + runCmd.Flags().String(logPathFlag, "", "Log file path") + _ = viper.BindPFlag(logPathFlag, runCmd.Flags().Lookup(logPathFlag)) + runCmd.Flags().String(endpointFlag, "", "Endpoint URL") + _ = viper.BindPFlag(endpointFlag, runCmd.Flags().Lookup(endpointFlag)) +} + +func logResponse(cmd *cobra.Command, id int, resp *http.Response, logReq request2.LoggedRequest) { + cmd.Println(strconv.Itoa(id)+")", logReq.Method, logReq.URI) + cmd.Println(resp.Status) + if resp.ContentLength != 0 { + body, err := io.ReadAll(io.LimitReader(resp.Body, resp.ContentLength)) + if err != nil { + cmd.Println(id, err) + } + cmd.Println(string(body)) + } + cmd.Println() +} + +func run(cmd *cobra.Command, _ []string) error { + ctx := request2.SetCredentials( + cmd.Context(), + viper.GetString(awsAccessKey), + viper.GetString(awsSecretKey), + ) + ctx = request2.WithMultiparts(ctx) + + cmd.SetContext(ctx) + + file, err := os.Open(viper.GetString(logPathFlag)) + if err != nil { + return err + } + defer file.Close() + + reader := bufio.NewReader(file) + + client := &http.Client{ + Transport: http.DefaultTransport, + Timeout: time.Second * time.Duration(viper.GetInt64(httpTimeout)), + } + + if viper.GetBool(skipVerifyTLS) { + client.Transport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + id := 1 + for { + req, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } + + return err + } + + var logReq request2.LoggedRequest + if err = json.Unmarshal([]byte(req), &logReq); err != nil { + cmd.PrintErrln(strconv.Itoa(id)+")", "failed to parse request", err) + id++ + continue + } + + select { + case <-ctx.Done(): + return fmt.Errorf("interrupted: %w", ctx.Err()) + default: + resp, err := playback(cmd, logReq, client) + if err != nil { + cmd.PrintErrln(strconv.Itoa(id)+")", "failed to playback request:", err) + id++ + continue + } + logResponse(cmd, id, resp, logReq) + id++ + } + } + + return nil +} + +// playback creates http.Request from LoggedRequest and sends it to specified endpoint. +func playback(cmd *cobra.Command, logReq request2.LoggedRequest, client *http.Client) (*http.Response, error) { + r, err := prepareRequest(cmd, logReq) + if err != nil { + return nil, fmt.Errorf("failed to prepare request: %w", err) + } + resp, err := client.Do(r) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + defer resp.Body.Close() + + if err = request2.HandleResponse(cmd.Context(), r, respBody, logReq.Response); err != nil { + return nil, fmt.Errorf("failed to register multipart upload: %w", err) + } + resp.Body = io.NopCloser(bytes.NewBuffer(respBody)) + + return resp, nil +} + +// prepareRequest creates request from logs and modifies its signature and uploadId (if presents). +func prepareRequest(cmd *cobra.Command, logReq request2.LoggedRequest) (*http.Request, error) { + r, err := http.NewRequest(logReq.Method, viper.GetString(endpointFlag)+logReq.URI, + bytes.NewReader(logReq.Body)) + if err != nil { + return nil, err + } + r.Header = logReq.Header + + err = request2.SwapUploadID(cmd.Context(), r) + if err != nil { + return nil, err + } + + sha256hash := sha256.New() + sha256hash.Write(logReq.Body) + r.Header.Set(auth.AmzContentSHA256, hex.EncodeToString(sha256hash.Sum(nil))) + if r.Header.Get(api.ContentMD5) != "" { + sha256hash.Reset() + md5hash := md5.New() + md5hash.Write(logReq.Body) + r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(md5hash.Sum(nil))) + } + err = request2.Sign(cmd, r) + if err != nil { + return nil, err + } + return r, nil +} diff --git a/cmd/s3-playback/request/multipart.go b/cmd/s3-playback/request/multipart.go new file mode 100644 index 0000000..8253cee --- /dev/null +++ b/cmd/s3-playback/request/multipart.go @@ -0,0 +1,74 @@ +package request + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "net/http" +) + +type MultipartUpload struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"` + Bucket string `json:"bucket" xml:"Bucket"` + Key string `json:"key" xml:"Key"` + UploadID string `json:"uploadId" xml:"UploadId"` +} + +func WithMultiparts(ctx context.Context) context.Context { + return context.WithValue(ctx, multipartKey{}, map[string]MultipartUpload{}) +} + +func SetMultipartUpload(ctx context.Context, oldUploadID string, upload MultipartUpload) error { + mparts, ok := ctx.Value(multipartKey{}).(map[string]MultipartUpload) + if !ok { + return errors.New("multiparts not set") + } + mparts[oldUploadID] = upload + + return nil +} + +func GetMultipart(ctx context.Context, oldUploadID string) (MultipartUpload, error) { + mparts, ok := ctx.Value(multipartKey{}).(map[string]MultipartUpload) + if !ok { + return MultipartUpload{}, errors.New("no multipart map set") + } + + return mparts[oldUploadID], nil +} + +func HandleResponse(ctx context.Context, r *http.Request, resp []byte, logResponse []byte) error { + var mpart, mpartOld MultipartUpload + if r.Method == "POST" && r.URL.Query().Has("uploads") { + err1 := xml.Unmarshal(resp, &mpart) + err2 := xml.Unmarshal(logResponse, &mpartOld) + if err1 != nil || err2 != nil { + return errors.New("xml unmarshal error") + } + if mpartOld.UploadID != "" { + if err := SetMultipartUpload(ctx, mpartOld.UploadID, mpart); err != nil { + return err + } + } + } + + return nil +} + +func SwapUploadID(ctx context.Context, r *http.Request) error { + var uploadID string + query := r.URL.Query() + + if query.Has("uploadId") { + uploadID = query.Get("uploadId") + mpart, err := GetMultipart(ctx, uploadID) + if err != nil { + return fmt.Errorf("failed to get multipart upload: %w", err) + } + query.Set("uploadId", mpart.UploadID) + r.URL.RawQuery = query.Encode() + } + + return nil +} diff --git a/cmd/s3-playback/request/request.go b/cmd/s3-playback/request/request.go new file mode 100644 index 0000000..89917dc --- /dev/null +++ b/cmd/s3-playback/request/request.go @@ -0,0 +1,101 @@ +package request + +import ( + "context" + "errors" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/spf13/cobra" +) + +// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter. +var authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P[^/]+)/(?P[^/]+)/(?P[^/]*)/(?P[^/]+)/aws4_request,\s*SignedHeaders=(?P.+),\s*Signature=(?P.+)`) + +type ( + LoggedRequest struct { + From string `json:"from"` + URI string `json:"URI"` + Method string `json:"method"` + Query url.Values `json:"query"` + Body []byte `json:"body"` + Header http.Header `json:"headers"` + Response []byte `json:"response"` + } + Credentials struct { + AccessKey string + SecretKey string + } + contextKey struct{} + multipartKey struct{} +) + +func SetCredentials(ctx context.Context, accessKey, secretKey string) context.Context { + return context.WithValue(ctx, contextKey{}, + Credentials{ + AccessKey: accessKey, SecretKey: secretKey, + }, + ) +} + +func GetCredentials(ctx context.Context) (Credentials, error) { + val, ok := ctx.Value(contextKey{}).(Credentials) + if !ok { + return val, errors.New("credentials not set") + } + + return val, nil +} + +// Sign replace Authorization header with new Access key id and Signature values. +func Sign(cmd *cobra.Command, r *http.Request) error { + ctx := cmd.Context() + creds, err := GetCredentials(ctx) + if err != nil { + return errors.New("failed to get credentials") + } + credProvider, err := credentials.NewStaticCredentialsProvider(creds.AccessKey, + creds.SecretKey, "").Retrieve(ctx) + if err != nil { + return err + } + + authInfo, err := parseAuthHeader(r) + if err != nil { + return err + } + r.Header[auth.AuthorizationHdr][0] = strings.Replace(r.Header[auth.AuthorizationHdr][0], + authInfo["access_key_id"], creds.AccessKey, 1) + + signer := v4.NewSigner() + signatureDateTimeStr := r.Header.Get(api.AmzDate) + signatureDateTime, err := time.Parse("20060102T150405Z", signatureDateTimeStr) + if err != nil { + return err + } + + err = signer.SignHTTP(ctx, credProvider, r, r.Header.Get(api.AmzContentSha256), + authInfo["service"], authInfo["region"], signatureDateTime) + if err != nil { + return err + } + + return nil +} + +func parseAuthHeader(r *http.Request) (map[string]string, error) { + matcher := auth.NewRegexpMatcher(authorizationFieldRegexp) + authHeader := r.Header[auth.AuthorizationHdr] + if len(authHeader) != 1 { + return nil, errors.New("invalid authorization header") + } + authinfo := matcher.GetSubmatches(authHeader[0]) + return authinfo, nil +} diff --git a/docs/playback.md b/docs/playback.md new file mode 100644 index 0000000..625b517 --- /dev/null +++ b/docs/playback.md @@ -0,0 +1,41 @@ +# FrostFS S3 PlayBack + +Playback is a tool to reproduce PoC network activity in dev environment. Network logs could be +gathered from s3-gw via HTTP Logger which could be enabled on build with `loghttp` build tag +and `http_logging` option enabled in app settings. + +## Configuration + +Playback accepts configuration file path in yaml with corresponding options: +```yaml +endpoint: http://localhost:8084 +log: ./request.log +env: .env +credentials: + access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD + secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30 +http_timeout: 360 +skip_verify_tls: true +``` +Configuration path is passed via required `--config` flag. + +### Configuration parameters + +#### Persistent params +| # | Parameter | Flag name | Type | Default value | Description | +|---|-----------------------|-----------------|--------|---------------|--------------------------------------------------------------------| +| 1 | http_timeout | http-timeout | int | 60 | http request timeout | +| 2 | skip_verify_tls | skip-verify-tls | bool | false | skips tls certificate verification for self-signed https endpoints | +| 3 | credentials.accessKey | - | string | - | aws access key id | +| 4 | credentials.secretKey | - | string | - | aws secret key | + +#### `run` command parameters +| # | Parameter | Flag name | Type | Default value | Description | +|---|-----------|-----------|--------|-----------------------|--------------------------------------------------------| +| 1 | endpoint | endpoint | string | http://localhost:8080 | s3-gw endpoint URL | +| 2 | log | log | string | ./request.log | path to log file, could be either absolute or relative | +| | | | | | | +| | | | | | | +| | | | | | | +| | | | | | | +| | | | | | | diff --git a/go.mod b/go.mod index 84b4588..2c5e2dc 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,8 @@ require ( git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/aws/aws-sdk-go v1.44.6 - github.com/aws/aws-sdk-go-v2 v1.18.1 + github.com/aws/aws-sdk-go-v2 v1.30.3 + github.com/aws/aws-sdk-go-v2/credentials v1.17.27 github.com/bluele/gcache v0.0.2 github.com/go-chi/chi/v5 v5.0.8 github.com/google/uuid v1.3.1 @@ -43,7 +44,7 @@ require ( git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect - github.com/aws/smithy-go v1.13.5 // indirect + github.com/aws/smithy-go v1.20.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/go.sum b/go.sum index d79d47b..04a4d7a 100644 --- a/go.sum +++ b/go.sum @@ -64,10 +64,12 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8 github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/aws/aws-sdk-go v1.44.6 h1:Y+uHxmZfhRTLX2X3khkdxCoTZAyGEX21aOUHe1U6geg= github.com/aws/aws-sdk-go v1.44.6/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= -github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= +github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4= +github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= +github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= @@ -172,7 +174,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=