diff --git a/api/handler/put.go b/api/handler/put.go index c3776f79..37796608 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -352,10 +352,13 @@ func (h *handler) getBodyReader(r *http.Request) (io.ReadCloser, error) { err error chunkReader io.ReadCloser ) - if shaType == api.StreamingContentV4aSHA256 { - chunkReader, err = newSignV4aChunkedReader(r) - } else { + switch shaType { + case api.StreamingContentSHA256, api.StreamingContentSHA256Trailer: chunkReader, err = newSignV4ChunkedReader(r) + case api.StreamingContentV4aSHA256, api.StreamingContentV4aSHA256Trailer: + chunkReader, err = newSignV4aChunkedReader(r) + default: + chunkReader, err = newUnsignedChunkedReader(r) } if err != nil { diff --git a/api/handler/put_test.go b/api/handler/put_test.go index bf8863cf..ffb2b41e 100644 --- a/api/handler/put_test.go +++ b/api/handler/put_test.go @@ -377,6 +377,46 @@ func TestPutObjectCheckContentSHA256(t *testing.T) { } } +func TestPutObjectWithStreamUnsignedBody(t *testing.T) { + hc := prepareHandlerContext(t) + + bktName, objName := "examplebucket", "chunkObject.txt" + createTestBucket(hc, bktName) + + w, req, chunk := getChunkedRequestTrailing(hc.context, t, bktName, objName) + hc.Handler().PutObjectHandler(w, req) + assertStatus(t, w, http.StatusOK) + + w, req = prepareTestRequest(hc, bktName, objName, nil) + hc.Handler().HeadObjectHandler(w, req) + assertStatus(t, w, http.StatusOK) + require.Equal(t, strconv.Itoa(awsChunkedRequestExampleDecodedContentLength), w.Header().Get(api.ContentLength)) + + data := getObjectRange(t, hc, bktName, objName, 0, awsChunkedRequestExampleDecodedContentLength) + for i := range chunk { + require.Equal(t, chunk[i], data[i]) + } +} + +func TestPutObjectWithStreamBodyAWSExampleTrailing(t *testing.T) { + hc := prepareHandlerContext(t) + + bktName, objName := "examplebucket", "chunkObject.txt" + createTestBucket(hc, bktName) + + w, req, chunk := getChunkedRequestUnsignedTrailing(hc.context, t, bktName, objName) + hc.Handler().PutObjectHandler(w, req) + assertStatus(t, w, http.StatusOK) + + w, req = prepareTestRequest(hc, bktName, objName, nil) + hc.Handler().HeadObjectHandler(w, req) + assertStatus(t, w, http.StatusOK) + require.Equal(t, strconv.Itoa(awsChunkedRequestExampleDecodedContentLength), w.Header().Get(api.ContentLength)) + + data := getObjectRange(t, hc, bktName, objName, 0, awsChunkedRequestExampleDecodedContentLength) + equalDataSlices(t, chunk, data) +} + func TestPutObjectWithStreamBodyAWSExample(t *testing.T) { hc := prepareHandlerContext(t) @@ -476,9 +516,9 @@ func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName strin req, err := http.NewRequest("PUT", "https://s3.amazonaws.com/"+bktName+"/"+objName, nil) require.NoError(t, err) - req.Header.Set("content-encoding", "aws-chunked") + req.Header.Set("content-encoding", api.AwsChunked) req.Header.Set("content-length", strconv.Itoa(awsChunkedRequestExampleContentLength)) - req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD") + req.Header.Set("x-amz-content-sha256", api.StreamingContentSHA256) req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength)) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") @@ -510,6 +550,133 @@ func getChunkedRequest(ctx context.Context, t *testing.T, bktName, objName strin return w, req, chunk } +// getChunkedRequestTrailing implements request example from +// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html +func getChunkedRequestTrailing(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) { + chunk := make([]byte, 65*1024) + for i := range chunk { + chunk[i] = 'a' + } + chunk1 := chunk[:64*1024] + chunk2 := chunk[64*1024:] + + AWSAccessKeyID := "AKIAIOSFODNN7EXAMPLE" + AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + + awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey} + signer := v4.NewSigner() + + reqBody := bytes.NewBufferString("10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n") + _, err := reqBody.Write(chunk1) + require.NoError(t, err) + _, err = reqBody.WriteString("\r\n400;chunk-signature=1c1344b170168f8e65b41376b44b20fe354e373826ccbbe2c1d40a8cae51e5c7\r\n") + require.NoError(t, err) + _, err = reqBody.Write(chunk2) + require.NoError(t, err) + _, err = reqBody.WriteString("\r\n0;chunk-signature=2ca2aba2005185cf7159c6277faf83795951dd77a3a99e6e65d5c9f85863f992\r\n") + require.NoError(t, err) + _, err = reqBody.WriteString("\r\nx-amz-checksum-crc32c:sOO8/Q==\r\n") + require.NoError(t, err) + _, err = reqBody.WriteString("\r\nx-amz-trailer-signature:63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f\r\n\r\n") + require.NoError(t, err) + + req, err := http.NewRequest("PUT", "https://s3.amazonaws.com/"+bktName+"/"+objName, nil) + require.NoError(t, err) + req.Header.Set("content-encoding", api.AwsChunked) + req.Header.Set("content-length", strconv.Itoa(awsChunkedRequestExampleContentLength)) + req.Header.Set("x-amz-content-sha256", api.StreamingContentSHA256Trailer) + req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength)) + req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32") + + signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z") + require.NoError(t, err) + + err = signer.SignHTTP(ctx, awsCreds, req, api.StreamingContentSHA256Trailer, "s3", "us-east-1", signTime) + require.NoError(t, err) + + req.Body = io.NopCloser(reqBody) + + w := httptest.NewRecorder() + reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "") + req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo)) + req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{ + ClientTime: signTime, + AuthHeaders: &middleware.AuthHeader{ + AccessKeyID: AWSAccessKeyID, + SignatureV4: "106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e", + Region: "us-east-1", + }, + AccessBox: &accessbox.Box{ + Gate: &accessbox.GateData{ + SecretKey: AWSSecretAccessKey, + }, + }, + })) + + return w, req, chunk +} + +func getChunkedRequestUnsignedTrailing(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) { + chunk := make([]byte, 65*1024) + for i := range chunk { + chunk[i] = 'a' + } + //chunk1 := chunk[:64*1024] + //chunk2 := chunk[64*1024:] + + AWSAccessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6" + AWSSecretAccessKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e" + + awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey} + signer := v4.NewSigner() + + reqBody := bytes.NewBufferString("10400\r\n") + _, err := reqBody.Write(chunk) + require.NoError(t, err) + _, err = reqBody.WriteString("\r\n0\r\n") + require.NoError(t, err) + _, err = reqBody.WriteString("\r\nx-amz-checksum-crc64nvme:pRf+emrnL+A=\r\n\r\n") + require.NoError(t, err) + + req, err := http.NewRequest("PUT", "https://localhost:8184/"+bktName+"/"+objName, nil) + //req, err := http.NewRequest("PUT", "https://localhost:8184/test2/body", nil) + require.NoError(t, err) + req.Header.Set("x-amz-sdk-checksum-algorithm", "CRC64NVME") + req.Header.Set("content-encoding", api.AwsChunked) + req.Header.Set("x-amz-trailer", "x-amz-checksum-crc64nvme") + req.Header.Set("x-amz-content-sha256", api.StreamingUnsignedPayloadTrailer) + req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength)) + req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + + signTime, err := time.Parse("20060102T150405Z", "20250131T140527Z") + require.NoError(t, err) + + err = signer.SignHTTP(ctx, awsCreds, req, api.StreamingContentSHA256Trailer, "s3", "ru", signTime) + require.NoError(t, err) + + req.Body = io.NopCloser(reqBody) + + w := httptest.NewRecorder() + reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktName, Object: objName}, "") + req = req.WithContext(middleware.SetReqInfo(ctx, reqInfo)) + req = req.WithContext(middleware.SetBox(req.Context(), &middleware.Box{ + ClientTime: signTime, + AuthHeaders: &middleware.AuthHeader{ + AccessKeyID: AWSAccessKeyID, + SignatureV4: "a075c83779d1c3c02254fbe4c9eff0a21556d15556fc6a25db69147c4838226b", + Region: "ru", + }, + AccessBox: &accessbox.Box{ + Gate: &accessbox.GateData{ + SecretKey: AWSSecretAccessKey, + }, + }, + })) + + return w, req, chunk +} + func getEmptyChunkedRequest(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) { AWSAccessKeyID := "48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh" AWSSecretAccessKey := "09260955b4eb0279dc017ba20a1ddac909cbd226c86cbb2d868e55534c8e64b0" diff --git a/api/handler/s3unsignedreader.go b/api/handler/s3unsignedreader.go new file mode 100644 index 00000000..f18ec0aa --- /dev/null +++ b/api/handler/s3unsignedreader.go @@ -0,0 +1,157 @@ +package handler + +import ( + "bufio" + "context" + "encoding/hex" + "io" + "net/http" + "time" + + v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4sdk2/signer/v4" + errs "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + "github.com/aws/aws-sdk-go-v2/aws" +) + +type ( + s3UnsignedChunkReader struct { + ctx context.Context + reader *bufio.Reader + streamSigner *v4.StreamSigner + + requestTime time.Time + buffer []byte + offset int + err error + } +) + +func (c *s3UnsignedChunkReader) Close() (err error) { + return nil +} + +func (c *s3UnsignedChunkReader) Read(buf []byte) (num int, err error) { + if c.offset > 0 { + num = copy(buf, c.buffer[c.offset:]) + if num == len(buf) { + c.offset += num + return num, nil + } + c.offset = 0 + buf = buf[num:] + } + + var size int + var b byte + for { + b, err = c.reader.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + c.err = err + return num, c.err + } + if b == '\r' { + break + } + + // Manually deserialize the size since AWS specified + // the chunk size to be of variable width. In particular, + // a size of 16 is encoded as `10` while a size of 64 KB + // is `10000`. + switch { + case b >= '0' && b <= '9': + size = size<<4 | int(b-'0') + case b >= 'a' && b <= 'f': + size = size<<4 | int(b-('a'-10)) + case b >= 'A' && b <= 'F': + size = size<<4 | int(b-('A'-10)) + default: + c.err = errMalformedChunkedEncoding + return num, c.err + } + if size > maxChunkSize { + c.err = errGiantChunk + return num, c.err + } + } + + if b != '\r' { + c.err = errMalformedChunkedEncoding + return num, c.err + } + b, err = c.reader.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + c.err = err + return num, c.err + } + if b != '\n' { + c.err = errMalformedChunkedEncoding + return num, c.err + } + + if cap(c.buffer) < size { + c.buffer = make([]byte, size) + } else { + c.buffer = c.buffer[:size] + } + + // Now, we read the payload and compute its SHA-256 hash. + _, err = io.ReadFull(c.reader, c.buffer) + if err == io.EOF && size != 0 { + err = io.ErrUnexpectedEOF + } + if err != nil && err != io.EOF { + c.err = err + return num, c.err + } + + // If the chunk size is zero we return io.EOF. As specified by AWS, + // only the last chunk is zero-sized. + if size == 0 { + c.err = io.EOF + return num, c.err + } + + c.offset = copy(buf, c.buffer) + num += c.offset + return num, err +} + +func newUnsignedChunkedReader(req *http.Request) (io.ReadCloser, error) { + ctx := req.Context() + box, err := middleware.GetBoxData(ctx) + if err != nil { + return nil, errs.GetAPIError(errs.ErrAuthorizationHeaderMalformed) + } + + authHeaders, err := middleware.GetAuthHeaders(ctx) + if err != nil { + return nil, errs.GetAPIError(errs.ErrAuthorizationHeaderMalformed) + } + + currentCredentials := aws.Credentials{AccessKeyID: authHeaders.AccessKeyID, SecretAccessKey: box.Gate.SecretKey} + seed, err := hex.DecodeString(authHeaders.SignatureV4) + if err != nil { + return nil, errs.GetAPIError(errs.ErrSignatureDoesNotMatch) + } + + reqTime, err := middleware.GetClientTime(ctx) + if err != nil { + return nil, errs.GetAPIError(errs.ErrMalformedDate) + } + newStreamSigner := v4.NewStreamSigner(currentCredentials, "s3", authHeaders.Region, seed) + + return &s3UnsignedChunkReader{ + ctx: ctx, + reader: bufio.NewReader(req.Body), + streamSigner: newStreamSigner, + requestTime: reqTime, + buffer: make([]byte, 64*1024), + }, nil +} diff --git a/api/headers.go b/api/headers.go index bfcb2952..c877b4b5 100644 --- a/api/headers.go +++ b/api/headers.go @@ -94,8 +94,11 @@ const ( DefaultLocationConstraint = "default" - StreamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" - StreamingContentV4aSHA256 = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD" + StreamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + StreamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" + StreamingContentV4aSHA256 = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD" + StreamingContentV4aSHA256Trailer = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER" + StreamingUnsignedPayloadTrailer = "STREAMING-UNSIGNED-PAYLOAD-TRAILER" DefaultStorageClass = "STANDARD" ) @@ -129,6 +132,8 @@ var SystemMetadata = map[string]struct{}{ func IsSignedStreamingV4(r *http.Request) (string, bool) { shaHeader := r.Header.Get(AmzContentSha256) return shaHeader, - (shaHeader == StreamingContentSHA256 || shaHeader == StreamingContentV4aSHA256) && + (shaHeader == StreamingContentSHA256 || shaHeader == StreamingContentSHA256Trailer || + shaHeader == StreamingContentV4aSHA256 || shaHeader == StreamingContentV4aSHA256Trailer || + shaHeader == StreamingUnsignedPayloadTrailer) && r.Method == http.MethodPut }