[#369] Request reproducer

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
Nikita Zinkevich 2024-08-01 11:21:20 +03:00
parent 6f87a9b3b6
commit 013e89f4a1
11 changed files with 618 additions and 6 deletions

View file

@ -0,0 +1,7 @@
endpoint: http://localhost:8084
log: ./log/multipart5.log
credentials:
access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD
secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30
http_timeout: 60s
skip_verify_tls: false

20
cmd/s3-playback/main.go Normal file
View file

@ -0,0 +1,20 @@
package main
import (
"context"
"os"
"os/signal"
"syscall"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/modules"
)
func main() {
ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
if cmd, err := modules.Execute(ctx); err != nil {
cmd.PrintErrln("Error:", err.Error())
cmd.PrintErrf("Run '%v --help' for usage.\n", cmd.CommandPath())
os.Exit(1)
}
}

View file

@ -0,0 +1,65 @@
package modules
import (
"context"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
const (
configPath = "config"
configPathFlag = "config"
httpTimeout = "http_timeout"
httpTimeoutFlag = "http-timeout"
skipVerifyTLS = "skip_verify_tls"
skipVerifyTLSFlag = "skip-verify-tls"
)
var (
cfgFile string
rootCmd = &cobra.Command{
Use: "frostfs-s3-playback",
Version: version.Version,
Short: "FrostFS S3 Traffic Playback",
Long: "Helps to reproduce s3 commands from log files",
Example: "frostfs-s3-playback [--skip-verify-tls] [--http-timeout <timeout>] " +
"[--version] --config <config_path> <command>",
SilenceUsage: true,
SilenceErrors: true,
PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
_ = viper.BindPFlag(configPath, cmd.PersistentFlags().Lookup(configPathFlag))
_ = viper.BindPFlag(httpTimeout, cmd.PersistentFlags().Lookup(httpTimeoutFlag))
_ = viper.BindPFlag(skipVerifyTLS, cmd.PersistentFlags().Lookup(skipVerifyTLSFlag))
return nil
},
RunE: func(cmd *cobra.Command, _ []string) error {
return cmd.Help()
},
}
)
func Execute(ctx context.Context) (*cobra.Command, error) {
return rootCmd.ExecuteContextC(ctx)
}
func initConfig() {
viper.SetConfigFile(cfgFile)
_ = viper.ReadInConfig()
}
func init() {
cobra.OnInitialize(initConfig)
rootCmd.PersistentFlags().StringVar(&cfgFile, configPathFlag, "", "configuration filepath")
_ = rootCmd.MarkPersistentFlagRequired(configPathFlag)
_ = rootCmd.MarkPersistentFlagFilename(configPathFlag)
rootCmd.PersistentFlags().Duration(httpTimeoutFlag, time.Minute, "http request timeout")
rootCmd.PersistentFlags().Bool(skipVerifyTLSFlag, false, "skip TLS certificate verification")
rootCmd.AddCommand(runCmd)
}

View file

@ -0,0 +1,194 @@
package modules
import (
"bufio"
"bytes"
"context"
"crypto/md5"
"crypto/sha256"
"crypto/tls"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"strconv"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/cmd/s3-playback/request"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
const (
logPathFlag = "log"
endpointFlag = "endpoint"
awsAccessKey = "credentials.access_key"
awsSecretKey = "credentials.secret_key"
)
var runCmd = &cobra.Command{
Use: "run",
Short: "Send requests from log file",
Long: "Reads the network log file and sends each request to the specified URL",
Example: "frostfs-s3-playback --config <config_path> run [--endpoint=<endpoint>] [--log=<log_path>]",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if rootCmd.PersistentPreRunE != nil {
if err := rootCmd.PersistentPreRunE(cmd, args); err != nil {
return err
}
}
_ = viper.BindPFlag(logPathFlag, cmd.Flags().Lookup(logPathFlag))
_ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag))
return nil
},
RunE: run,
}
func init() {
runCmd.Flags().String(logPathFlag, "./request.log", "log file path")
runCmd.Flags().String(endpointFlag, "", "endpoint URL")
}
func logResponse(cmd *cobra.Command, id int, resp *http.Response, logReq request.LoggedRequest) {
cmd.Println(strconv.Itoa(id)+")", logReq.Method, logReq.URI)
cmd.Println(resp.Status)
if resp.ContentLength != 0 {
body, err := io.ReadAll(io.LimitReader(resp.Body, resp.ContentLength))
if err != nil {
cmd.Println(id, err)
}
cmd.Println(string(body))
}
cmd.Println()
}
func run(cmd *cobra.Command, _ []string) error {
ctx := request.SetCredentials(
cmd.Context(),
viper.GetString(awsAccessKey),
viper.GetString(awsSecretKey),
)
ctx = request.WithMultiparts(ctx)
file, err := os.Open(viper.GetString(logPathFlag))
if err != nil {
return err
}
defer file.Close()
reader := bufio.NewReader(file)
client := &http.Client{
Transport: http.DefaultTransport,
Timeout: viper.GetDuration(httpTimeout),
}
if viper.GetBool(skipVerifyTLS) {
client.Transport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
id := 1
for {
logReq, err := getRequestFromLog(reader)
if err != nil {
if err == io.EOF {
break
}
cmd.PrintErrln(strconv.Itoa(id)+")", "failed to parse request", err)
id++
continue
}
select {
case <-ctx.Done():
return fmt.Errorf("interrupted: %w", ctx.Err())
default:
resp, err := playback(ctx, logReq, client)
if err != nil {
cmd.PrintErrln(strconv.Itoa(id)+")", "failed to playback request:", err)
id++
continue
}
logResponse(cmd, id, resp, logReq)
id++
}
}
return nil
}
func getRequestFromLog(reader *bufio.Reader) (request.LoggedRequest, error) {
var logReq request.LoggedRequest
req, err := reader.ReadString('\n')
if err != nil {
return logReq, err
}
err = json.Unmarshal([]byte(req), &logReq)
if err != nil {
return logReq, err
}
return logReq, nil
}
// playback creates http.Request from LoggedRequest and sends it to specified endpoint.
func playback(ctx context.Context, logReq request.LoggedRequest, client *http.Client) (*http.Response, error) {
r, err := prepareRequest(ctx, logReq)
if err != nil {
return nil, fmt.Errorf("failed to prepare request: %w", err)
}
resp, err := client.Do(r)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body: %w", err)
}
defer resp.Body.Close()
if err = request.HandleResponse(ctx, r, respBody, logReq.Response); err != nil {
return nil, fmt.Errorf("failed to register multipart upload: %w", err)
}
resp.Body = io.NopCloser(bytes.NewBuffer(respBody))
return resp, nil
}
// prepareRequest creates request from logs and modifies its signature and uploadId (if presents).
func prepareRequest(ctx context.Context, logReq request.LoggedRequest) (*http.Request, error) {
r, err := http.NewRequestWithContext(ctx, logReq.Method, viper.GetString(endpointFlag)+logReq.URI,
bytes.NewReader(logReq.Body))
if err != nil {
return nil, err
}
r.Header = logReq.Header
err = request.SwapUploadID(ctx, r)
if err != nil {
return nil, err
}
sha256hash := sha256.New()
sha256hash.Write(logReq.Body)
r.Header.Set(auth.AmzContentSHA256, hex.EncodeToString(sha256hash.Sum(nil)))
if r.Header.Get(api.ContentMD5) != "" {
sha256hash.Reset()
md5hash := md5.New()
md5hash.Write(logReq.Body)
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(md5hash.Sum(nil)))
}
err = request.Sign(ctx, r)
if err != nil {
return nil, err
}
return r, nil
}

View file

@ -0,0 +1,74 @@
package request
import (
"context"
"encoding/xml"
"errors"
"fmt"
"net/http"
)
type MultipartUpload struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
Bucket string `json:"bucket" xml:"Bucket"`
Key string `json:"key" xml:"Key"`
UploadID string `json:"uploadId" xml:"UploadId"`
}
func WithMultiparts(ctx context.Context) context.Context {
return context.WithValue(ctx, multipartKey{}, map[string]MultipartUpload{})
}
func SetMultipartUpload(ctx context.Context, oldUploadID string, upload MultipartUpload) error {
mparts, ok := ctx.Value(multipartKey{}).(map[string]MultipartUpload)
if !ok {
return errors.New("multiparts not set")
}
mparts[oldUploadID] = upload
return nil
}
func GetMultipart(ctx context.Context, oldUploadID string) (MultipartUpload, error) {
mparts, ok := ctx.Value(multipartKey{}).(map[string]MultipartUpload)
if !ok {
return MultipartUpload{}, errors.New("no multipart map set")
}
return mparts[oldUploadID], nil
}
func HandleResponse(ctx context.Context, r *http.Request, resp []byte, logResponse []byte) error {
var mpart, mpartOld MultipartUpload
if r.Method == "POST" && r.URL.Query().Has("uploads") {
err1 := xml.Unmarshal(resp, &mpart)
err2 := xml.Unmarshal(logResponse, &mpartOld)
if err1 != nil || err2 != nil {
return errors.New("xml unmarshal error")
}
if mpartOld.UploadID != "" {
if err := SetMultipartUpload(ctx, mpartOld.UploadID, mpart); err != nil {
return err
}
}
}
return nil
}
func SwapUploadID(ctx context.Context, r *http.Request) error {
var uploadID string
query := r.URL.Query()
if query.Has("uploadId") {
uploadID = query.Get("uploadId")
mpart, err := GetMultipart(ctx, uploadID)
if err != nil {
return fmt.Errorf("failed to get multipart upload: %w", err)
}
query.Set("uploadId", mpart.UploadID)
r.URL.RawQuery = query.Encode()
}
return nil
}

View file

@ -0,0 +1,101 @@
package request
import (
"context"
"errors"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/credentials"
)
var (
// authorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter.
authorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P<access_key_id>[^/]+)/(?P<date>[^/]+)/(?P<region>[^/]*)/(?P<service>[^/]+)/aws4_request,\s*SignedHeaders=(?P<signed_header_fields>.+),\s*Signature=(?P<v4_signature>.+)`)
ErrNoMatches = errors.New("no matches found")
)
type (
LoggedRequest struct {
From string `json:"from"`
URI string `json:"URI"`
Method string `json:"method"`
Query url.Values `json:"query"`
Body []byte `json:"body"`
Header http.Header `json:"headers"`
Response []byte `json:"response"`
}
Credentials struct {
AccessKey string
SecretKey string
}
contextKey struct{}
multipartKey struct{}
)
func SetCredentials(ctx context.Context, accessKey, secretKey string) context.Context {
return context.WithValue(ctx, contextKey{},
Credentials{
AccessKey: accessKey, SecretKey: secretKey,
},
)
}
func GetCredentials(ctx context.Context) (Credentials, error) {
val, ok := ctx.Value(contextKey{}).(Credentials)
if !ok {
return val, errors.New("credentials not set")
}
return val, nil
}
// Sign replace Authorization header with new Access key id and Signature values.
func Sign(ctx context.Context, r *http.Request) error {
creds, err := GetCredentials(ctx)
if err != nil {
return errors.New("failed to get credentials")
}
credProvider, err := credentials.NewStaticCredentialsProvider(creds.AccessKey,
creds.SecretKey, "").Retrieve(ctx)
if err != nil {
return err
}
authInfo, err := parseAuthHeader(r.Header.Get(auth.AuthorizationHdr))
if err != nil {
return err
}
r.Header[auth.AuthorizationHdr][0] = strings.Replace(r.Header[auth.AuthorizationHdr][0],
authInfo["access_key_id"], creds.AccessKey, 1)
signer := v4.NewSigner()
signatureDateTimeStr := r.Header.Get(api.AmzDate)
signatureDateTime, err := time.Parse("20060102T150405Z", signatureDateTimeStr)
if err != nil {
return err
}
err = signer.SignHTTP(ctx, credProvider, r, r.Header.Get(api.AmzContentSha256),
authInfo["service"], authInfo["region"], signatureDateTime)
if err != nil {
return err
}
return nil
}
func parseAuthHeader(authHeader string) (map[string]string, error) {
authInfo := auth.NewRegexpMatcher(authorizationFieldRegexp).GetSubmatches(authHeader)
if len(authInfo) == 0 {
return nil, ErrNoMatches
}
return authInfo, nil
}

View file

@ -0,0 +1,95 @@
package request
import (
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func withoutValue(data map[string]string, field string) map[string]string {
result := make(map[string]string)
for k, v := range data {
result[k] = v
}
result[field] = ""
return result
}
func TestParseAuthHeader(t *testing.T) {
defaultHeader := "AWS4-HMAC-SHA256 Credential=oid0cid/20210809/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2811ccb9e242f41426738fb1f"
defaultAuthInfo := map[string]string{
"access_key_id": "oid0cid",
"service": "s3",
"region": "us-east-1",
"v4_signature": "2811ccb9e242f41426738fb1f",
"signed_header_fields": "host;x-amz-content-sha256;x-amz-date",
"date": "20210809",
}
for _, tc := range []struct {
title string
header string
err error
expected map[string]string
}{
{
title: "correct full header",
header: defaultHeader,
err: nil,
expected: defaultAuthInfo,
},
{
title: "correct with empty region",
header: strings.Replace(defaultHeader, "/us-east-1/", "//", 1),
err: nil,
expected: withoutValue(defaultAuthInfo, "region"),
},
{
title: "empty access key",
header: strings.Replace(defaultHeader, "oid0cid", "", 1),
err: ErrNoMatches,
expected: nil,
},
{
title: "empty service",
header: strings.Replace(defaultHeader, "/s3/", "//", 1),
err: ErrNoMatches,
expected: nil,
},
{
title: "empty date",
header: strings.Replace(defaultHeader, "/20210809/", "//", 1),
err: ErrNoMatches,
expected: nil,
},
{
title: "empty v4_signature",
header: strings.Replace(defaultHeader, "Signature=2811ccb9e242f41426738fb1f",
"Signature=", 1),
err: ErrNoMatches,
expected: nil,
},
{
title: "empty signed_fields",
header: strings.Replace(defaultHeader, "SignedHeaders=host;x-amz-content-sha256;x-amz-date",
"SignedHeaders=", 1),
err: ErrNoMatches,
expected: nil,
},
{
title: "empty signed_fields",
header: strings.Replace(defaultHeader, "SignedHeaders=host;x-amz-content-sha256;x-amz-date",
"SignedHeaders=", 1),
err: ErrNoMatches,
expected: nil,
},
} {
t.Run(tc.title, func(t *testing.T) {
authInfo, err := parseAuthHeader(tc.header)
require.ErrorIs(t, err, tc.err, tc.header)
require.Equal(t, tc.expected, authInfo, tc.header)
})
}
}

View file

@ -380,6 +380,11 @@ logger:
### `http_logging` section
Could be enabled only in builds with `loghttp` build tag. To build with `loghttp` tag, pass `GOFLAGS` var to `make`:
```bash
make GOFLAGS="-tags=loghttp" [target]
```
```yaml
http_logging:
enabled: false

47
docs/playback.md Normal file
View file

@ -0,0 +1,47 @@
# FrostFS S3 Playback
Playback is a tool to reproduce queries to `frostfs-s3-gw` in dev environment. Network logs could be
gathered from `s3-gw` via HTTP Logger which could be enabled on build with `loghttp` build tag
and `http_logging.enabled` option set to `true` in `s3-gw` configuration.
## Commands
`run` - reads log file and reproduces send requests from it to specified endpoint
#### Example
```bash
frostfs-s3-playback --config <config_path> run [--endpoint=<endpoint>] [--log=<log_path>]
```
## Configuration
Playback accepts configuration file path in yaml with corresponding options:
```yaml
endpoint: http://localhost:8084
log: ./request.log
env: .env
credentials:
access_key: CAtUxDSSFtuVyVCjHTMhwx3eP3YSPo5ffwbPcnKfcdrD06WwUSn72T5EBNe3jcgjL54rmxFM6u3nUAoNBps8qJ1PD
secret_key: 560027d81c277de7378f71cbf12a32e4f7f541de724be59bcfdbfdc925425f30
http_timeout: 60s
skip_verify_tls: true
```
Configuration path is passed via required `--config` flag.
If corresponding flag is set, it overrides parameter from config.
### Configuration parameters
#### Global params
| # | Config parameter name | Flag name | Type | Default value | Description |
|---|-----------------------|-----------------|----------|---------------|--------------------------------------------------------|
| 1 | - | config | string | - | config file path (e.g. `./config/playback.yaml`) |
| 2 | http_timeout | http-timeout | duration | 60s | http request timeout |
| 3 | skip_verify_tls | skip-verify-tls | bool | false | skips tls certificate verification for https endpoints |
| 4 | credentials.accessKey | - | string | - | AWS access key id |
| 5 | credentials.secretKey | - | string | - | AWS secret key |
#### `run` command parameters
| # | Config parameter name | Flag name | Type | Default value | Description |
|---|-----------------------|-----------|--------|---------------|--------------------------------------------------------|
| 1 | endpoint | endpoint | string | - | s3-gw endpoint URL |
| 2 | log | log | string | ./request.log | path to log file, could be either absolute or relative |

5
go.mod
View file

@ -10,7 +10,8 @@ require (
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240821072038-a1386f6d259a
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
github.com/aws/aws-sdk-go v1.44.6
github.com/aws/aws-sdk-go-v2 v1.18.1
github.com/aws/aws-sdk-go-v2 v1.30.3
github.com/aws/aws-sdk-go-v2/credentials v1.17.27
github.com/bluele/gcache v0.0.2
github.com/go-chi/chi/v5 v5.0.8
github.com/google/uuid v1.6.0
@ -45,7 +46,7 @@ require (
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
github.com/VictoriaMetrics/easyproto v0.1.4 // indirect
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/aws/smithy-go v1.20.3 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect

11
go.sum
View file

@ -66,10 +66,12 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
github.com/aws/aws-sdk-go v1.44.6 h1:Y+uHxmZfhRTLX2X3khkdxCoTZAyGEX21aOUHe1U6geg=
github.com/aws/aws-sdk-go v1.44.6/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo=
github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI=
github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
@ -177,6 +179,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=