From 711d6b2c71d0ccf534a440658d5d1acc615e17a4 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 13 Feb 2025 17:01:03 +0300 Subject: [PATCH] [#642] Simplify tests Signed-off-by: Denis Kirillov --- api/auth/center.go | 26 +-- api/auth/center_test.go | 2 +- api/auth/presign_test.go | 2 +- api/handler/put_test.go | 339 ++++++++++++++++++++------------------- api/headers.go | 1 + 5 files changed, 192 insertions(+), 178 deletions(-) diff --git a/api/auth/center.go b/api/auth/center.go index 99570274..df15d82e 100644 --- a/api/auth/center.go +++ b/api/auth/center.go @@ -33,8 +33,8 @@ var ( // AuthorizationFieldRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter. AuthorizationFieldRegexp = regexp.MustCompile(`AWS4-HMAC-SHA256 Credential=(?P[^/]+)/(?P[^/]+)/(?P[^/]*)/(?P[^/]+)/aws4_request,\s*SignedHeaders=(?P.+),\s*Signature=(?P.+)`) - // authorizationFieldV4aRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter. - authorizationFieldV4aRegexp = regexp.MustCompile(`AWS4-ECDSA-P256-SHA256 Credential=(?P[^/]+)/(?P[^/]+)/(?P[^/]+)/aws4_request,\s*SignedHeaders=(?P.+),\s*Signature=(?P.+)`) + // AuthorizationFieldV4aRegexp -- is regexp for credentials with Base58 encoded cid and oid and '0' (zero) as delimiter. + AuthorizationFieldV4aRegexp = regexp.MustCompile(`AWS4-ECDSA-P256-SHA256 Credential=(?P[^/]+)/(?P[^/]+)/(?P[^/]+)/aws4_request,\s*SignedHeaders=(?P.+),\s*Signature=(?P.+)`) // postPolicyCredentialRegexp -- is regexp for credentials when uploading file using POST with policy. postPolicyCredentialRegexp = regexp.MustCompile(`(?P[^/]+)/(?P[^/]+)/(?P[^/]*)/(?P[^/]+)/aws4_request`) @@ -107,7 +107,7 @@ func New(creds tokens.Credentials, prefixes []string, settings CenterSettings) * return &Center{ cli: creds, reg: NewRegexpMatcher(AuthorizationFieldRegexp), - regV4a: NewRegexpMatcher(authorizationFieldV4aRegexp), + regV4a: NewRegexpMatcher(AuthorizationFieldV4aRegexp), postReg: NewRegexpMatcher(postPolicyCredentialRegexp), allowedAccessKeyIDPrefixes: prefixes, settings: settings, @@ -115,8 +115,8 @@ func New(creds tokens.Credentials, prefixes []string, settings CenterSettings) * } const ( - signaturePreambleSigV4 = "AWS4-HMAC-SHA256" - signaturePreambleSigV4A = "AWS4-ECDSA-P256-SHA256" + SignaturePreambleSigV4 = "AWS4-HMAC-SHA256" + SignaturePreambleSigV4A = "AWS4-ECDSA-P256-SHA256" ) func (c *Center) parseAuthHeader(authHeader string, headers http.Header) (*AuthHeader, error) { @@ -128,13 +128,13 @@ func (c *Center) parseAuthHeader(authHeader string, headers http.Header) (*AuthH ) switch preamble { - case signaturePreambleSigV4: + case SignaturePreambleSigV4: submatches = c.reg.GetSubmatches(authHeader) if len(submatches) != authHeaderPartsNum { return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), authHeader) } region = submatches["region"] - case signaturePreambleSigV4A: + case SignaturePreambleSigV4A: submatches = c.regV4a.GetSubmatches(authHeader) if len(submatches) != authHeaderV4aPartsNum { return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), authHeader) @@ -170,7 +170,7 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) { ) queryValues := r.URL.Query() - if queryValues.Get(AmzAlgorithm) == signaturePreambleSigV4 { + if queryValues.Get(AmzAlgorithm) == SignaturePreambleSigV4 { creds := strings.Split(queryValues.Get(AmzCredential), "/") if len(creds) != 5 || creds[4] != "aws4_request" { return nil, fmt.Errorf("bad X-Amz-Credential") @@ -183,7 +183,7 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) { SignedFields: strings.Split(queryValues.Get(AmzSignedHeaders), ";"), Date: creds[1], IsPresigned: true, - Preamble: signaturePreambleSigV4, + Preamble: SignaturePreambleSigV4, PayloadHash: r.Header.Get(AmzContentSHA256), } authHdr.Expiration, err = time.ParseDuration(queryValues.Get(AmzExpires) + "s") @@ -191,7 +191,7 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) { return nil, fmt.Errorf("%w: couldn't parse X-Amz-Expires %v", apierr.GetAPIError(apierr.ErrMalformedExpires), err) } signatureDateTimeStr = queryValues.Get(AmzDate) - } else if queryValues.Get(AmzAlgorithm) == signaturePreambleSigV4A { + } else if queryValues.Get(AmzAlgorithm) == SignaturePreambleSigV4A { creds := strings.Split(queryValues.Get(AmzCredential), "/") if len(creds) != 4 || creds[3] != "aws4_request" { return nil, fmt.Errorf("bad X-Amz-Credential") @@ -204,7 +204,7 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) { SignedFields: strings.Split(queryValues.Get(AmzSignedHeaders), ";"), Date: creds[1], IsPresigned: true, - Preamble: signaturePreambleSigV4A, + Preamble: SignaturePreambleSigV4A, PayloadHash: r.Header.Get(AmzContentSHA256), } authHdr.Expiration, err = time.ParseDuration(queryValues.Get(AmzExpires) + "s") @@ -402,7 +402,7 @@ func (c *Center) checkSign(ctx context.Context, authHeader *AuthHeader, box *acc } switch authHeader.Preamble { - case signaturePreambleSigV4: + case SignaturePreambleSigV4: creds := aws.Credentials{ AccessKeyID: authHeader.AccessKeyID, SecretAccessKey: box.Gate.SecretKey, @@ -437,7 +437,7 @@ func (c *Center) checkSign(ctx context.Context, authHeader *AuthHeader, box *acc authHeader.Signature, signature, authHeader.SignedFields) } - case signaturePreambleSigV4A: + case SignaturePreambleSigV4A: signer := v4a.NewSigner(func(options *v4a.SignerOptions) { options.DisableURIPathEscaping = true }) diff --git a/api/auth/center_test.go b/api/auth/center_test.go index e12d1c8c..9bbf02ed 100644 --- a/api/auth/center_test.go +++ b/api/auth/center_test.go @@ -69,7 +69,7 @@ func TestAuthHeaderParse(t *testing.T) { Signature: "2811ccb9e242f41426738fb1f", SignedFields: []string{"host", "x-amz-content-sha256", "x-amz-date"}, Date: "20210809", - Preamble: signaturePreambleSigV4, + Preamble: SignaturePreambleSigV4, }, }, { diff --git a/api/auth/presign_test.go b/api/auth/presign_test.go index 361a7e45..7f738518 100644 --- a/api/auth/presign_test.go +++ b/api/auth/presign_test.go @@ -145,7 +145,7 @@ func TestCheckSignV4a(t *testing.T) { c := &Center{ cli: mock, - regV4a: NewRegexpMatcher(authorizationFieldV4aRegexp), + regV4a: NewRegexpMatcher(AuthorizationFieldV4aRegexp), postReg: NewRegexpMatcher(postPolicyCredentialRegexp), } box, err := c.Authenticate(req) diff --git a/api/handler/put_test.go b/api/handler/put_test.go index de22bbf8..5178efce 100644 --- a/api/handler/put_test.go +++ b/api/handler/put_test.go @@ -10,6 +10,8 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" + "hash/crc32" "io" "mime/multipart" "net/http" @@ -426,7 +428,7 @@ func TestPutObjectWithStreamBodyAWSExampleTrailing(t *testing.T) { createTestBucket(hc, bktName) t.Run("valid trailer signature", func(t *testing.T) { - w, req, chunk := getChunkedRequestTrailing(hc.context, t, bktName, objName) + w, req, chunk := getChunkedRequestAWSExampleTrailing(t, bktName, objName) hc.Handler().PutObjectHandler(w, req) assertStatus(t, w, http.StatusOK) @@ -440,7 +442,7 @@ func TestPutObjectWithStreamBodyAWSExampleTrailing(t *testing.T) { }) t.Run("invalid trailer signature", func(t *testing.T) { - w, req, _ := getChunkedRequestTrailing(hc.context, t, bktName, objName) + w, req, _ := getChunkedRequestAWSExampleTrailing(t, bktName, objName) body := req.Body.(*customNopCloser) body.Bytes()[body.Len()-2] = 'a' hc.Handler().PutObjectHandler(w, req) @@ -454,7 +456,7 @@ func TestPutObjectWithStreamBodyAWSExample(t *testing.T) { bktName, objName := "examplebucket", "chunkObject.txt" createTestBucket(hc, bktName) - w, req, chunk := getChunkedRequest(hc.context, t, bktName, objName) + w, req, chunk := getChunkedRequestAWSExample(t, bktName, objName) hc.Handler().PutObjectHandler(w, req) assertStatus(t, w, http.StatusOK) @@ -469,13 +471,17 @@ func TestPutObjectWithStreamBodyAWSExample(t *testing.T) { } } -func TestPutObjectWithStreamEmptyBodyAWSExample(t *testing.T) { +func TestPutObjectWithStreamEmptyBodyAWSExampleWithContentType(t *testing.T) { hc := prepareHandlerContext(t) bktName, objName := "dkirillov", "tmp" createTestBucket(hc, bktName) - w, req := getEmptyChunkedRequest(hc.context, t, bktName, objName) + signTime, err := time.Parse("20060102T150405Z", "20241003T100055Z") + require.NoError(t, err) + + extra := [2]string{api.ContentType, "text/plain; charset=UTF-8"} + w, req := getChunkedRequestBase(t, bktName, objName, nil, api.StreamingContentSHA256, signTime, extra) hc.Handler().PutObjectHandler(w, req) assertStatus(t, w, http.StatusOK) @@ -495,11 +501,14 @@ func TestPutObjectWithStreamEmptyBody(t *testing.T) { bktName := "bucket" createTestBucket(hc, bktName) + signTime, err := time.Parse("20060102T150405Z", "20241003T100055Z") + require.NoError(t, err) + t.Run("unsigned", func(t *testing.T) { t.Run("trailer", func(t *testing.T) { objName := "unsigned trailer" - w, req := getEmptyChunkedRequestUnsigned(hc.context, t, bktName, objName) + w, req := getEmptyChunkedRequestUnsigned(t, bktName, objName) req.Header.Del(api.ContentType) hc.Handler().PutObjectHandler(w, req) assertStatus(t, w, http.StatusOK) @@ -511,10 +520,23 @@ func TestPutObjectWithStreamEmptyBody(t *testing.T) { }) t.Run("sigv4", func(t *testing.T) { + t.Run("trailer", func(t *testing.T) { + objName := "sigv4 trailer" + + w, req := getChunkedRequestBase(t, bktName, objName, nil, api.StreamingContentSHA256Trailer, signTime) + req.Header.Del(api.ContentType) + hc.Handler().PutObjectHandler(w, req) + assertStatus(t, w, http.StatusOK) + + d, h := getObject(hc, bktName, objName) + require.Empty(t, d) + require.Equal(t, "0", h.Get(api.ContentLength)) + }) + t.Run("no trailer", func(t *testing.T) { objName := "sigv4 no trailer" - w, req := getEmptyChunkedRequest(hc.context, t, bktName, objName) + w, req := getChunkedRequestBase(t, bktName, objName, nil, api.StreamingContentSHA256, signTime) req.Header.Del(api.ContentType) hc.Handler().PutObjectHandler(w, req) assertStatus(t, w, http.StatusOK) @@ -529,7 +551,20 @@ func TestPutObjectWithStreamEmptyBody(t *testing.T) { t.Run("trailer", func(t *testing.T) { objName := "sigv4a trailer" - w, req := getEmptyChunkedRequestSigv4a(hc.context, t, bktName, objName) + w, req := getEmptyChunkedRequestSigv4aWithTrailers(t, bktName, objName) + req.Header.Del(api.ContentType) + hc.Handler().PutObjectHandler(w, req) + assertStatus(t, w, http.StatusOK) + + d, h := getObject(hc, bktName, objName) + require.Empty(t, d) + require.Equal(t, "0", h.Get(api.ContentLength)) + }) + + t.Run("no trailer", func(t *testing.T) { + objName := "sigv4a no trailer" + + w, req := getEmptyChunkedRequestSigv4a(t, bktName, objName) req.Header.Del(api.ContentType) hc.Handler().PutObjectHandler(w, req) assertStatus(t, w, http.StatusOK) @@ -547,7 +582,7 @@ func TestPutChunkedTestContentEncoding(t *testing.T) { bktName, objName := "examplebucket", "chunkObject.txt" createTestBucket(hc, bktName) - w, req, _ := getChunkedRequest(hc.context, t, bktName, objName) + w, req, _ := getChunkedRequestAWSExample(t, bktName, objName) req.Header.Set(api.ContentEncoding, api.AwsChunked+",gzip") hc.Handler().PutObjectHandler(w, req) @@ -556,13 +591,13 @@ func TestPutChunkedTestContentEncoding(t *testing.T) { resp := headObjectBase(hc, bktName, objName, emptyVersion) require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding)) - w, req, _ = getChunkedRequest(hc.context, t, bktName, objName) + w, req, _ = getChunkedRequestAWSExample(t, bktName, objName) req.Header.Set(api.ContentEncoding, "gzip") hc.Handler().PutObjectHandler(w, req) assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidEncodingMethod)) hc.config.bypassContentEncodingInChunks = true - w, req, _ = getChunkedRequest(hc.context, t, bktName, objName) + w, req, _ = getChunkedRequestAWSExample(t, bktName, objName) req.Header.Set(api.ContentEncoding, "gzip") hc.Handler().PutObjectHandler(w, req) assertStatus(t, w, http.StatusOK) @@ -571,9 +606,9 @@ func TestPutChunkedTestContentEncoding(t *testing.T) { require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding)) } -// getChunkedRequest implements request example from +// getChunkedRequestAWSExample implements request example from // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html -func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) { +func getChunkedRequestAWSExample(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) { chunk := make([]byte, 65*1024) for i := range chunk { chunk[i] = 'a' @@ -581,12 +616,8 @@ func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName strin chunk1 := chunk[:64*1024] chunk2 := chunk[64*1024:] - AWSAccessKeyID := "AKIAIOSFODNN7EXAMPLE" AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey} - signer := v4.NewSigner() - reqBody := bytes.NewBufferString("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n") _, err := reqBody.Write(chunk1) require.NoError(t, err) @@ -604,32 +635,14 @@ func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName strin req.Header.Set("x-amz-content-sha256", api.StreamingContentSHA256) req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength)) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9") signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z") require.NoError(t, err) - err = signer.SignHTTP(ctx, awsCreds, req, auth.UnsignedPayload, "s3", "us-east-1", signTime) - require.NoError(t, err) - req.Body = io.NopCloser(reqBody) - w := httptest.NewRecorder() - reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "") - req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo)) - req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{ - ClientTime: signTime, - AuthHeaders: &middleware.AuthHeader{ - AccessKeyID: AWSAccessKeyID, - SignatureV4: "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9", - Region: "us-east-1", - }, - AccessBox: &accessbox.Box{ - Gate: &accessbox.GateData{ - SecretKey: AWSSecretAccessKey, - }, - }, - })) - + w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey) return w, req, chunk } @@ -641,9 +654,9 @@ func (c *customNopCloser) Close() error { return nil } -// getChunkedRequestTrailing implements request example from +// getChunkedRequestAWSExampleTrailing implements request example from // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html -func getChunkedRequestTrailing(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) { +func getChunkedRequestAWSExampleTrailing(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) { chunk := make([]byte, 65*1024) for i := range chunk { chunk[i] = 'a' @@ -651,12 +664,8 @@ func getChunkedRequestTrailing(ctx context.Context, t *testing.T, bktName, objNa chunk1 := chunk[:64*1024] chunk2 := chunk[64*1024:] - AWSAccessKeyID := "AKIAIOSFODNN7EXAMPLE" AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey} - signer := v4.NewSigner() - reqBody := bytes.NewBufferString("10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n") _, err := reqBody.Write(chunk1) require.NoError(t, err) @@ -686,32 +695,14 @@ func getChunkedRequestTrailing(ctx context.Context, t *testing.T, bktName, objNa req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength)) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32c") + req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e") signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z") require.NoError(t, err) - err = signer.SignHTTP(ctx, awsCreds, req, api.StreamingContentSHA256Trailer, "s3", "us-east-1", signTime) - require.NoError(t, err) - req.Body = &customNopCloser{Buffer: reqBody} - w := httptest.NewRecorder() - reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "") - req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo)) - req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{ - ClientTime: signTime, - AuthHeaders: &middleware.AuthHeader{ - AccessKeyID: AWSAccessKeyID, - SignatureV4: "106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e", - Region: "us-east-1", - }, - AccessBox: &accessbox.Box{ - Gate: &accessbox.GateData{ - SecretKey: AWSSecretAccessKey, - }, - }, - })) - + w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey) return w, req, chunk } @@ -720,8 +711,6 @@ func getChunkedRequestUnsignedTrailing(ctx context.Context, t *testing.T, bktNam for i := range chunk { chunk[i] = 'a' } - //chunk1 := chunk[:64*1024] - //chunk2 := chunk[64*1024:] AWSAccessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6" AWSSecretAccessKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e" @@ -738,7 +727,6 @@ func getChunkedRequestUnsignedTrailing(ctx context.Context, t *testing.T, bktNam require.NoError(t, err) req, err := http.NewRequest("PUT", "https://localhost:8184/"+bktName+"/"+objName, nil) - //req, err := http.NewRequest("PUT", "https://localhost:8184/test2/body", nil) require.NoError(t, err) req.Header.Set("x-amz-sdk-checksum-algorithm", "CRC64NVME") req.Header.Set("content-encoding", api.AwsChunked) @@ -755,23 +743,7 @@ func getChunkedRequestUnsignedTrailing(ctx context.Context, t *testing.T, bktNam req.Body = io.NopCloser(reqBody) - w := httptest.NewRecorder() - reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "") - req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo)) - req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{ - ClientTime: signTime, - AuthHeaders: &middleware.AuthHeader{ - AccessKeyID: AWSAccessKeyID, - SignatureV4: "a075c83779d1c3c02254fbe4c9eff0a21556d15556fc6a25db69147c4838226b", - Region: "ru", - }, - AccessBox: &accessbox.Box{ - Gate: &accessbox.GateData{ - SecretKey: AWSSecretAccessKey, - }, - }, - })) - + w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey) return w, req, chunk } @@ -809,69 +781,113 @@ func getChunkedRequestUnsignedTrailingSmall(ctx context.Context, t *testing.T, b req.Body = io.NopCloser(reqBody) - w := httptest.NewRecorder() - reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "") - req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo)) - req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{ - ClientTime: signTime, - AuthHeaders: &middleware.AuthHeader{ - AccessKeyID: AWSAccessKeyID, - SignatureV4: "a075c83779d1c3c02254fbe4c9eff0a21556d15556fc6a25db69147c4838226b", - Region: "ru", - }, - AccessBox: &accessbox.Box{ - Gate: &accessbox.GateData{ - SecretKey: AWSSecretAccessKey, - }, - }, - })) - + w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey) return w, req, []byte(chunk) } -func getEmptyChunkedRequest(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) { - AWSAccessKeyID := "48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh" - AWSSecretAccessKey := "09260955b4eb0279dc017ba20a1ddac909cbd226c86cbb2d868e55534c8e64b0" +func getChunkedRequestBase(t *testing.T, bktName, objName string, chunks [][]byte, shaType string, signTime time.Time, extraHeaders ...[2]string) (*httptest.ResponseRecorder, *http.Request) { + creds := aws.Credentials{ + AccessKeyID: "48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh", + SecretAccessKey: "09260955b4eb0279dc017ba20a1ddac909cbd226c86cbb2d868e55534c8e64b0", + } + region := "us-east-1" + service := "s3" - reqBody := bytes.NewBufferString("0;chunk-signature=311a7142c8f3a07972c3aca65c36484b513a8fee48ab7178c7225388f2ae9894\r\n\r\n") - - req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, reqBody) - require.NoError(t, err) - req.Header.Set("Amz-Sdk-Invocation-Id", "8a8cd4be-aef8-8034-f08d-a6144ade41f9") - req.Header.Set("Amz-Sdk-Request", "attempt=1; max=2") - req.Header.Set(api.Authorization, "AWS4-HMAC-SHA256 Credential=48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh/20241003/us-east-1/s3/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-encoding;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length, Signature=4b530ab4af2381f214941af591266b209968264a2c94337fa1efc048c7dff352") - req.Header.Set(api.ContentEncoding, "aws-chunked") - req.Header.Set(api.ContentLength, "86") - req.Header.Set(api.ContentType, "text/plain; charset=UTF-8") - req.Header.Set(api.AmzDate, "20241003T100055Z") - req.Header.Set(api.AmzContentSha256, "STREAMING-AWS4-HMAC-SHA256-PAYLOAD") - req.Header.Set(api.AmzDecodedContentLength, "0") - - signTime, err := time.Parse("20060102T150405Z", "20241003T100055Z") + req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, nil) require.NoError(t, err) - w := httptest.NewRecorder() - reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "") - req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo)) - req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{ + payloadLength := 0 + for _, chunk := range chunks { + payloadLength += len(chunk) + } + + for _, kv := range extraHeaders { + req.Header.Set(kv[0], kv[1]) + } + + req.Header.Set(api.ContentEncoding, api.AwsChunked) + req.Header.Set(api.AmzDecodedContentLength, strconv.Itoa(payloadLength)) + req.Header.Set(api.AmzDate, signTime.Format("20060102T150405Z")) + req.Header.Set(api.AmzContentSha256, shaType) + if shaType == api.StreamingContentSHA256Trailer { + req.Header.Set(api.AmzTrailer, "x-amz-checksum-crc32") + } + + signer := v4.NewSigner() + err = signer.SignHTTP(req.Context(), creds, req, shaType, service, region, signTime) + require.NoError(t, err) + + seedSignature := strings.Split(req.Header.Get(api.Authorization), "Signature=")[1] + seed, err := hex.DecodeString(seedSignature) + require.NoError(t, err) + + var reqBody bytes.Buffer + + hash := crc32.NewIEEE() + + newStreamSigner := v4.NewStreamSigner(creds, service, region, seed) + for _, chunk := range chunks { + _, err = hash.Write(chunk) + require.NoError(t, err) + + signature, err := newStreamSigner.GetSignature(req.Context(), nil, chunk, signTime) + require.NoError(t, err) + reqBody.WriteString(fmt.Sprintf("%x;chunk-signature=%x\r\n", len(chunk), signature)) + reqBody.Write(chunk) + reqBody.WriteString("\r\n") + } + signature, err := newStreamSigner.GetSignature(req.Context(), nil, nil, signTime) + require.NoError(t, err) + reqBody.WriteString(fmt.Sprintf("0;chunk-signature=%x\r\n", signature)) + + if shaType == api.StreamingContentSHA256Trailer { + crc32Res := hash.Sum(nil) + checksumStr := "x-amz-checksum-crc32:" + base64.StdEncoding.EncodeToString(crc32Res) + reqBody.WriteString(fmt.Sprintf("%s\r\n", checksumStr)) + trailerSignature, err := newStreamSigner.GetTrailerSignature([]byte(checksumStr+"\n"), signTime) + require.NoError(t, err) + reqBody.WriteString(fmt.Sprintf("x-amz-trailer-signature:%x\r\n", trailerSignature)) + } + reqBody.WriteString("\r\n") + + req.Body = io.NopCloser(&reqBody) + + return prepareReqMiddlewares(req, signTime, creds.SecretAccessKey) +} + +func prepareReqMiddlewares(req *http.Request, signTime time.Time, secretAccessKey string) (*httptest.ResponseRecorder, *http.Request) { + authHeader := req.Header.Get(api.Authorization) + var parsed map[string]string + var region string + if strings.HasPrefix(authHeader, auth.SignaturePreambleSigV4) { + parsed = auth.NewRegexpMatcher(auth.AuthorizationFieldRegexp).GetSubmatches(authHeader) + region = parsed["region"] + } else { + parsed = auth.NewRegexpMatcher(auth.AuthorizationFieldV4aRegexp).GetSubmatches(authHeader) + region = req.Header.Get("X-Amz-Region-Set") + } + + bktObj := strings.Split(req.URL.Path, "/") + + box := &middleware.Box{ ClientTime: signTime, AuthHeaders: &middleware.AuthHeader{ - AccessKeyID: AWSAccessKeyID, - SignatureV4: "4b530ab4af2381f214941af591266b209968264a2c94337fa1efc048c7dff352", - Region: "us-east-1", + AccessKeyID: parsed["access_key_id"], + SignatureV4: parsed["v4_signature"], + Region: region, }, - AccessBox: &accessbox.Box{ - Gate: &accessbox.GateData{ - SecretKey: AWSSecretAccessKey, - }, - }, - })) + AccessBox: &accessbox.Box{Gate: &accessbox.GateData{SecretKey: secretAccessKey}}, + } + + w := httptest.NewRecorder() + reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktObj[1], Object: bktObj[2]}, "") + req = req.WithContext(middleware.SetReqInfo(req.Context(), reqInfo)) + req = req.WithContext(middleware.SetBox(req.Context(), box)) return w, req } -func getEmptyChunkedRequestUnsigned(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) { - AWSAccessKeyID := "3jNrmDtHtuj1uLcixaSMA4KNUhNYhv1EpUNdFnbTXgUP071pGdSZfHSLtoC8gzjF5HoD6sC3Scq33t1WvvEvjmPnt" +func getEmptyChunkedRequestUnsigned(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) { AWSSecretAccessKey := "f1a0d650b650149f1a83140418e88a3c5572a0103e912e326492a91c19c4488a" reqBody := bytes.NewBufferString("0\r\nx-amz-checksum-crc64nvme:AAAAAAAAAAA=\r\n\r\n") @@ -889,24 +905,10 @@ func getEmptyChunkedRequestUnsigned(ctx context.Context, t *testing.T, bktName, signTime, err := time.Parse("20060102T150405Z", req.Header.Get(api.AmzDate)) require.NoError(t, err) - w := httptest.NewRecorder() - reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "") - req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo)) - req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{ - ClientTime: signTime, - AuthHeaders: &middleware.AuthHeader{ - AccessKeyID: AWSAccessKeyID, - SignatureV4: "1231b012c0ac313770c5a95ccf77b95b6c9b1c3760d6aa24cb8309801d56eb4a", - Region: "ru", - }, - AccessBox: &accessbox.Box{Gate: &accessbox.GateData{SecretKey: AWSSecretAccessKey}}, - })) - - return w, req + return prepareReqMiddlewares(req, signTime, AWSSecretAccessKey) } -func getEmptyChunkedRequestSigv4a(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) { - AWSAccessKeyID := "3jNrmDtHtuj1uLcixaSMA4KNUhNYhv1EpUNdFnbTXgUP071pGdSZfHSLtoC8gzjF5HoD6sC3Scq33t1WvvEvjmPnt" +func getEmptyChunkedRequestSigv4aWithTrailers(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) { AWSSecretAccessKey := "f1a0d650b650149f1a83140418e88a3c5572a0103e912e326492a91c19c4488a" body := "0;chunk-signature=3046022100ab9229a80d70f4d004768992881821a441a4ad4102e18de567e68216659bf497022100ec47a7a445351683557eedf893e6ed250c97af4b0415814671770b83766d69be\r\n" + @@ -924,27 +926,38 @@ func getEmptyChunkedRequestSigv4a(ctx context.Context, t *testing.T, bktName, ob req.Header.Set(api.AmzDecodedContentLength, "0") req.Header.Set(api.ContentLength, "367") req.Header.Set(api.ContentType, "text/plain: charset=UTF-8") - req.Header.Set("X-Amz-Region-Set", "use-east-1") + req.Header.Set("X-Amz-Region-Set", "us-east-1") req.Header.Set("X-Amz-Trailer", "x-amz-checksum-crc32") req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC32") signTime, err := time.Parse("20060102T150405Z", req.Header.Get(api.AmzDate)) require.NoError(t, err) - w := httptest.NewRecorder() - reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "") - req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo)) - req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{ - ClientTime: signTime, - AuthHeaders: &middleware.AuthHeader{ - AccessKeyID: AWSAccessKeyID, - SignatureV4: "304402202e1f1efcc56c588d9a94a3d8f20368686df8bfd5e8aad01fc4eff569ff38f1800220215198e3f1ba785492fe6703c4722872909ce8a09e8c9a13da90a9230c7a24b7", - Region: "us-east-1", - }, - AccessBox: &accessbox.Box{Gate: &accessbox.GateData{SecretKey: AWSSecretAccessKey}}, - })) + return prepareReqMiddlewares(req, signTime, AWSSecretAccessKey) +} - return w, req +func getEmptyChunkedRequestSigv4a(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) { + AWSSecretAccessKey := "f1a0d650b650149f1a83140418e88a3c5572a0103e912e326492a91c19c4488a" + + body := "0;chunk-signature=304502203f7c598a2e9a6673bf1ca30f5f6bebd0d76a4e9d3c16531448e96c2cda22d16a0221009e7ed578da0a9781366f1461a1484e64f15707f26d4310e59514db6ff9f7e0f1**\r\n\r\n" + + req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, bytes.NewBufferString(body)) + require.NoError(t, err) + req.Header.Set(api.Authorization, "AWS4-ECDSA-P256-SHA256 Credential=3jNrmDtHtuj1uLcixaSMA4KNUhNYhv1EpUNdFnbTXgUP071pGdSZfHSLtoC8gzjF5HoD6sC3Scq33t1WvvEvjmPnt/20250213/s3/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-region-set, Signature=3046022100dc589ea513448b996809db4b314a0b8a4a775c1165c6203c7104b2f1aae1243c0221009bf3a256e7c33415eaad20c1dbfb4e14cb00b362758bc4d2aaf94ca96a5f13f9") + req.Header.Set("Amz-Sdk-Invocation-Id", "f0814a40-0d74-066f-d01f-ed14f28ebfa4") + req.Header.Set("Amz-Sdk-Request", "attempt=1; max=2") + req.Header.Set(api.ContentEncoding, api.AwsChunked) + req.Header.Set(api.AmzDate, "20250213T135717Z") + req.Header.Set(api.AmzContentSha256, api.StreamingContentV4aSHA256) + req.Header.Set(api.AmzDecodedContentLength, "0") + req.Header.Set(api.ContentLength, "166") + req.Header.Set(api.ContentType, "text/plain: charset=UTF-8") + req.Header.Set("X-Amz-Region-Set", "use-east-1") + + signTime, err := time.Parse("20060102T150405Z", req.Header.Get(api.AmzDate)) + require.NoError(t, err) + + return prepareReqMiddlewares(req, signTime, AWSSecretAccessKey) } func TestCreateBucket(t *testing.T) { diff --git a/api/headers.go b/api/headers.go index c877b4b5..2316b77e 100644 --- a/api/headers.go +++ b/api/headers.go @@ -63,6 +63,7 @@ const ( AmzPartNumberMarker = "X-Amz-Part-Number-Marker" AmzStorageClass = "X-Amz-Storage-Class" AmzForceBucketDelete = "X-Amz-Force-Delete-Bucket" + AmzTrailer = "X-Amz-Trailer" AmzServerSideEncryptionCustomerAlgorithm = "x-amz-server-side-encryption-customer-algorithm" AmzServerSideEncryptionCustomerKey = "x-amz-server-side-encryption-customer-key"