forked from TrueCloudLab/frostfs-s3-gw
1415 lines
46 KiB
Go
1415 lines
46 KiB
Go
package handler
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/md5"
|
|
"crypto/rand"
|
|
"crypto/tls"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"hash/crc32"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
|
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4sdk2/signer/v4"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
const (
|
|
awsChunkedRequestExampleDecodedContentLength = 66560
|
|
awsChunkedRequestExampleContentLength = 66824
|
|
)
|
|
|
|
func TestCheckBucketName(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
err bool
|
|
}{
|
|
{name: "bucket"},
|
|
{name: "2bucket"},
|
|
{name: "buc-ket"},
|
|
{name: "abc"},
|
|
{name: "63aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
|
|
{name: "buc.ket", err: true},
|
|
{name: "buc.-ket", err: true},
|
|
{name: "bucket.", err: true},
|
|
{name: ".bucket", err: true},
|
|
{name: "bucket.", err: true},
|
|
{name: "bucket-", err: true},
|
|
{name: "-bucket", err: true},
|
|
{name: "Bucket", err: true},
|
|
{name: "buc.-ket", err: true},
|
|
{name: "buc-.ket", err: true},
|
|
{name: "Bucket", err: true},
|
|
{name: "buc!ket", err: true},
|
|
{name: "buc_ket", err: true},
|
|
{name: "xn--bucket", err: true},
|
|
{name: "bucket-s3alias", err: true},
|
|
{name: "192.168.0.1", err: true},
|
|
{name: "as", err: true},
|
|
{name: "64aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", err: true},
|
|
} {
|
|
err := checkBucketName(tc.name)
|
|
if tc.err {
|
|
require.Error(t, err, "bucket name: %s", tc.name)
|
|
} else {
|
|
require.NoError(t, err, "bucket name: %s", tc.name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCustomJSONMarshal(t *testing.T) {
|
|
data := []byte(`
|
|
{ "expiration": "2015-12-30T12:00:00.000Z",
|
|
"conditions": [
|
|
["content-length-range", 1048576, 10485760],
|
|
{"bucket": "bucketName"},
|
|
["starts-with", "$key", "user/user1/"]
|
|
]
|
|
}`)
|
|
|
|
parsedTime, err := time.Parse(time.RFC3339, "2015-12-30T12:00:00.000Z")
|
|
require.NoError(t, err)
|
|
|
|
expectedPolicy := &postPolicy{
|
|
Expiration: parsedTime,
|
|
Conditions: []*policyCondition{
|
|
{
|
|
Matching: "content-length-range",
|
|
Key: "1048576",
|
|
Value: "10485760",
|
|
},
|
|
{
|
|
Matching: "eq",
|
|
Key: "bucket",
|
|
Value: "bucketName",
|
|
},
|
|
{
|
|
Matching: "starts-with",
|
|
Key: "key",
|
|
Value: "user/user1/",
|
|
},
|
|
},
|
|
}
|
|
|
|
policy := &postPolicy{}
|
|
err = json.Unmarshal(data, policy)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, expectedPolicy, policy)
|
|
}
|
|
|
|
func TestEmptyPostPolicy(t *testing.T) {
|
|
r := &http.Request{
|
|
MultipartForm: &multipart.Form{
|
|
Value: map[string][]string{
|
|
"key": {"some-key"},
|
|
},
|
|
},
|
|
}
|
|
reqInfo := &middleware.ReqInfo{}
|
|
metadata := make(map[string]string)
|
|
|
|
_, err := checkPostPolicy(r, reqInfo, metadata)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
// if content length is greater than this value
|
|
// data will be writen to file location.
|
|
const maxContentSizeForFormData = 10
|
|
|
|
func TestPostObject(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
ns, bktName := "", "bucket"
|
|
createTestBucket(hc, bktName)
|
|
|
|
for _, tc := range []struct {
|
|
key string
|
|
filename string
|
|
content string
|
|
objName string
|
|
err bool
|
|
}{
|
|
{
|
|
key: "user/user1/${filename}",
|
|
filename: "object",
|
|
content: "content",
|
|
objName: "user/user1/object",
|
|
},
|
|
{
|
|
key: "user/user1/${filename}",
|
|
filename: "object",
|
|
content: "maxContentSizeForFormData",
|
|
objName: "user/user1/object",
|
|
},
|
|
{
|
|
key: "user/user1/key-object",
|
|
filename: "object",
|
|
content: "",
|
|
objName: "user/user1/key-object",
|
|
},
|
|
{
|
|
key: "user/user1/key-object",
|
|
filename: "object",
|
|
content: "maxContentSizeForFormData",
|
|
objName: "user/user1/key-object",
|
|
},
|
|
{
|
|
key: "",
|
|
filename: "object",
|
|
content: "",
|
|
objName: "object",
|
|
},
|
|
{
|
|
key: "",
|
|
filename: "object",
|
|
content: "maxContentSizeForFormData",
|
|
objName: "object",
|
|
},
|
|
{
|
|
// RFC 7578, Section 4.2 requires that if a filename is provided, the
|
|
// directory path information must not be used.
|
|
key: "",
|
|
filename: "dir/object",
|
|
content: "content",
|
|
objName: "object",
|
|
},
|
|
{
|
|
key: "object",
|
|
filename: "",
|
|
content: "content",
|
|
objName: "object",
|
|
},
|
|
{
|
|
key: "",
|
|
filename: "",
|
|
err: true,
|
|
},
|
|
} {
|
|
t.Run(tc.key+";"+tc.filename, func(t *testing.T) {
|
|
w := postObjectBase(hc, ns, bktName, tc.key, tc.filename, tc.content)
|
|
if tc.err {
|
|
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInternalError))
|
|
return
|
|
}
|
|
assertStatus(hc.t, w, http.StatusNoContent)
|
|
content, _ := getObject(hc, bktName, tc.objName)
|
|
require.Equal(t, tc.content, string(content))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPutObjectOverrideCopiesNumber(t *testing.T) {
|
|
tc := prepareHandlerContext(t)
|
|
|
|
bktName, objName := "bucket-for-copies-number", "object-for-copies-number"
|
|
bktInfo := createTestBucket(tc, bktName)
|
|
|
|
w, r := prepareTestRequest(tc, bktName, objName, nil)
|
|
r.Header.Set(api.MetadataPrefix+strings.ToUpper(layer.AttributeFrostfsCopiesNumber), "1")
|
|
tc.Handler().PutObjectHandler(w, r)
|
|
|
|
p := &layer.HeadObjectParams{
|
|
BktInfo: bktInfo,
|
|
Object: objName,
|
|
}
|
|
|
|
objInfo, err := tc.Layer().GetObjectInfo(tc.Context(), p)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "1", objInfo.Headers[layer.AttributeFrostfsCopiesNumber])
|
|
}
|
|
|
|
func TestPutObjectWithNegativeContentLength(t *testing.T) {
|
|
tc := prepareHandlerContext(t)
|
|
|
|
bktName, objName := "bucket-for-put", "object-for-put"
|
|
createTestBucket(tc, bktName)
|
|
|
|
content := []byte("content")
|
|
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
|
r.ContentLength = -1
|
|
tc.Handler().PutObjectHandler(w, r)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
w, r = prepareTestRequest(tc, bktName, objName, nil)
|
|
tc.Handler().HeadObjectHandler(w, r)
|
|
assertStatus(t, w, http.StatusOK)
|
|
require.Equal(t, strconv.Itoa(len(content)), w.Header().Get(api.ContentLength))
|
|
|
|
result := listVersions(t, tc, bktName)
|
|
require.Len(t, result.Version, 1)
|
|
require.EqualValues(t, len(content), result.Version[0].Size)
|
|
}
|
|
|
|
func TestPutObjectWithStreamBodyError(t *testing.T) {
|
|
tc := prepareHandlerContext(t)
|
|
|
|
bktName, objName := "bucket-for-put", "object-for-put"
|
|
createTestBucket(tc, bktName)
|
|
|
|
content := []byte("content")
|
|
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
|
r.Header.Set(api.AmzContentSha256, api.StreamingContentSHA256)
|
|
r.Header.Set(api.ContentEncoding, api.AwsChunked)
|
|
tc.Handler().PutObjectHandler(w, r)
|
|
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrMissingContentLength))
|
|
|
|
checkNotFound(t, tc, bktName, objName, emptyVersion)
|
|
}
|
|
|
|
func TestPutObjectWithInvalidContentMD5(t *testing.T) {
|
|
tc := prepareHandlerContext(t)
|
|
tc.config.md5Enabled = true
|
|
|
|
bktName, objName := "bucket-for-put", "object-for-put"
|
|
createTestBucket(tc, bktName)
|
|
|
|
content := []byte("content")
|
|
md5HeaderContent := make([]byte, md5.Size)
|
|
n, err := rand.Read(md5HeaderContent)
|
|
require.Equal(t, md5.Size, n)
|
|
require.NoError(t, err)
|
|
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
|
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(md5HeaderContent))
|
|
tc.Handler().PutObjectHandler(w, r)
|
|
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrBadDigest))
|
|
|
|
w, r = prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
|
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString([]byte("invalid")))
|
|
tc.Handler().PutObjectHandler(w, r)
|
|
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
|
|
|
w, r = prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
|
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString([]byte("")))
|
|
tc.Handler().PutObjectHandler(w, r)
|
|
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
|
|
|
|
checkNotFound(t, tc, bktName, objName, emptyVersion)
|
|
}
|
|
|
|
func TestPutObjectWithEnabledMD5(t *testing.T) {
|
|
tc := prepareHandlerContext(t)
|
|
tc.config.md5Enabled = true
|
|
|
|
bktName, objName := "bucket-for-put", "object-for-put"
|
|
createTestBucket(tc, bktName)
|
|
|
|
content := []byte("content")
|
|
md5Hash := md5.New()
|
|
md5Hash.Write(content)
|
|
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
|
tc.Handler().PutObjectHandler(w, r)
|
|
require.Equal(t, data.Quote(hex.EncodeToString(md5Hash.Sum(nil))), w.Header().Get(api.ETag))
|
|
}
|
|
|
|
func TestPutObjectCheckContentSHA256(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
bktName, objName := "bucket-for-put", "object-for-put"
|
|
createTestBucket(hc, bktName)
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
hash string
|
|
content []byte
|
|
error bool
|
|
}{
|
|
{
|
|
name: "invalid hash value",
|
|
hash: "d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
|
|
content: []byte("content"),
|
|
error: true,
|
|
},
|
|
{
|
|
name: "correct hash for empty payload",
|
|
hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
|
content: []byte(""),
|
|
error: false,
|
|
},
|
|
{
|
|
name: "unsigned payload",
|
|
hash: "UNSIGNED-PAYLOAD",
|
|
content: []byte("content"),
|
|
error: false,
|
|
},
|
|
{
|
|
name: "correct hash",
|
|
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
|
|
content: []byte("content"),
|
|
error: false,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
w, r := prepareTestPayloadRequest(hc, bktName, objName, bytes.NewReader(tc.content))
|
|
r.Header.Set("X-Amz-Content-Sha256", tc.hash)
|
|
hc.Handler().PutObjectHandler(w, r)
|
|
|
|
if tc.error {
|
|
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch))
|
|
|
|
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
|
hc.Handler().GetObjectHandler(w, r)
|
|
|
|
assertStatus(t, w, http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
assertStatus(t, w, http.StatusOK)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPutObjectWithStreamUnsignedBodySmall(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
bktName, objName := "test2", "tmp.txt"
|
|
createTestBucket(hc, bktName)
|
|
|
|
w, req, chunk := getChunkedRequestUnsignedTrailingSmall(hc.context, t, bktName, objName)
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
w, req = prepareTestRequest(hc, bktName, objName, nil)
|
|
hc.Handler().HeadObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
require.Equal(t, "5", w.Header().Get(api.ContentLength))
|
|
|
|
data := getObjectRange(t, hc, bktName, objName, 0, 5)
|
|
for i := range chunk {
|
|
require.Equal(t, chunk[i], data[i])
|
|
}
|
|
}
|
|
|
|
func TestPutObjectWithStreamUnsignedBody(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
bktName, objName := "examplebucket", "chunkObject.txt"
|
|
createTestBucket(hc, bktName)
|
|
|
|
w, req, chunk := getChunkedRequestUnsignedTrailing(hc.context, t, bktName, objName)
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
w, req = prepareTestRequest(hc, bktName, objName, nil)
|
|
hc.Handler().HeadObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
require.Equal(t, strconv.Itoa(awsChunkedRequestExampleDecodedContentLength), w.Header().Get(api.ContentLength))
|
|
|
|
data := getObjectRange(t, hc, bktName, objName, 0, awsChunkedRequestExampleDecodedContentLength)
|
|
for i := range chunk {
|
|
require.Equal(t, chunk[i], data[i])
|
|
}
|
|
}
|
|
|
|
func TestPutObjectWithStreamBodyAWSExampleTrailing(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
bktName, objName := "examplebucket", "chunkObject.txt"
|
|
createTestBucket(hc, bktName)
|
|
|
|
t.Run("valid trailer signature", func(t *testing.T) {
|
|
w, req, chunk := getChunkedRequestAWSExampleTrailing(t, bktName, objName)
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
w, req = prepareTestRequest(hc, bktName, objName, nil)
|
|
hc.Handler().HeadObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
require.Equal(t, strconv.Itoa(awsChunkedRequestExampleDecodedContentLength), w.Header().Get(api.ContentLength))
|
|
|
|
data := getObjectRange(t, hc, bktName, objName, 0, awsChunkedRequestExampleDecodedContentLength)
|
|
equalDataSlices(t, chunk, data)
|
|
})
|
|
|
|
t.Run("invalid trailer signature", func(t *testing.T) {
|
|
w, req, _ := getChunkedRequestAWSExampleTrailing(t, bktName, objName)
|
|
body := req.Body.(*customNopCloser)
|
|
body.Bytes()[body.Len()-2] = 'a'
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusForbidden)
|
|
})
|
|
}
|
|
|
|
func TestPutObjectWithStreamBodyAWSExample(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
bktName, objName := "examplebucket", "chunkObject.txt"
|
|
createTestBucket(hc, bktName)
|
|
|
|
w, req, chunk := getChunkedRequestAWSExample(t, bktName, objName)
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
w, req = prepareTestRequest(hc, bktName, objName, nil)
|
|
hc.Handler().HeadObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
require.Equal(t, strconv.Itoa(awsChunkedRequestExampleDecodedContentLength), w.Header().Get(api.ContentLength))
|
|
|
|
data := getObjectRange(t, hc, bktName, objName, 0, awsChunkedRequestExampleDecodedContentLength)
|
|
for i := range chunk {
|
|
require.Equal(t, chunk[i], data[i])
|
|
}
|
|
}
|
|
|
|
func TestPutObjectWithStreamEmptyBodyAWSExampleWithContentType(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
bktName, objName := "dkirillov", "tmp"
|
|
createTestBucket(hc, bktName)
|
|
|
|
signTime, err := time.Parse("20060102T150405Z", "20241003T100055Z")
|
|
require.NoError(t, err)
|
|
|
|
extra := [2]string{api.ContentType, "text/plain; charset=UTF-8"}
|
|
w, req := getChunkedRequestBase(t, bktName, objName, nil, api.StreamingContentSHA256, signTime, extra)
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
w, req = prepareTestRequest(hc, bktName, objName, nil)
|
|
hc.Handler().HeadObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
require.Equal(t, "0", w.Header().Get(api.ContentLength))
|
|
|
|
res := listObjectsV1(hc, bktName, "", "", "", -1)
|
|
require.Len(t, res.Contents, 1)
|
|
require.Empty(t, res.Contents[0].Size)
|
|
}
|
|
|
|
func TestPutObjectWithStreamEmptyBody(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
bktName := "bucket"
|
|
createTestBucket(hc, bktName)
|
|
|
|
signTime, err := time.Parse("20060102T150405Z", "20241003T100055Z")
|
|
require.NoError(t, err)
|
|
|
|
t.Run("unsigned", func(t *testing.T) {
|
|
t.Run("trailer", func(t *testing.T) {
|
|
objName := "unsigned trailer"
|
|
|
|
w, req := getEmptyChunkedRequestUnsigned(t, bktName, objName)
|
|
req.Header.Del(api.ContentType)
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
d, h := getObject(hc, bktName, objName)
|
|
require.Empty(t, d)
|
|
require.Equal(t, "0", h.Get(api.ContentLength))
|
|
})
|
|
})
|
|
|
|
t.Run("sigv4", func(t *testing.T) {
|
|
t.Run("trailer", func(t *testing.T) {
|
|
objName := "sigv4 trailer"
|
|
|
|
w, req := getChunkedRequestBase(t, bktName, objName, nil, api.StreamingContentSHA256Trailer, signTime)
|
|
req.Header.Del(api.ContentType)
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
d, h := getObject(hc, bktName, objName)
|
|
require.Empty(t, d)
|
|
require.Equal(t, "0", h.Get(api.ContentLength))
|
|
})
|
|
|
|
t.Run("no trailer", func(t *testing.T) {
|
|
objName := "sigv4 no trailer"
|
|
|
|
w, req := getChunkedRequestBase(t, bktName, objName, nil, api.StreamingContentSHA256, signTime)
|
|
req.Header.Del(api.ContentType)
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
d, h := getObject(hc, bktName, objName)
|
|
require.Empty(t, d)
|
|
require.Equal(t, "0", h.Get(api.ContentLength))
|
|
})
|
|
})
|
|
|
|
t.Run("sigv4a", func(t *testing.T) {
|
|
t.Run("trailer", func(t *testing.T) {
|
|
objName := "sigv4a trailer"
|
|
|
|
w, req := getEmptyChunkedRequestSigv4aWithTrailers(t, bktName, objName)
|
|
req.Header.Del(api.ContentType)
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
d, h := getObject(hc, bktName, objName)
|
|
require.Empty(t, d)
|
|
require.Equal(t, "0", h.Get(api.ContentLength))
|
|
})
|
|
|
|
t.Run("no trailer", func(t *testing.T) {
|
|
objName := "sigv4a no trailer"
|
|
|
|
w, req := getEmptyChunkedRequestSigv4a(t, bktName, objName)
|
|
req.Header.Del(api.ContentType)
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
d, h := getObject(hc, bktName, objName)
|
|
require.Empty(t, d)
|
|
require.Equal(t, "0", h.Get(api.ContentLength))
|
|
})
|
|
})
|
|
}
|
|
|
|
func TestPutChunkedTestContentEncoding(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
bktName, objName := "examplebucket", "chunkObject.txt"
|
|
createTestBucket(hc, bktName)
|
|
|
|
w, req, _ := getChunkedRequestAWSExample(t, bktName, objName)
|
|
req.Header.Set(api.ContentEncoding, api.AwsChunked+",gzip")
|
|
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
resp := headObjectBase(hc, bktName, objName, emptyVersion)
|
|
require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding))
|
|
|
|
w, req, _ = getChunkedRequestAWSExample(t, bktName, objName)
|
|
req.Header.Set(api.ContentEncoding, "gzip")
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidEncodingMethod))
|
|
|
|
hc.config.bypassContentEncodingInChunks = true
|
|
w, req, _ = getChunkedRequestAWSExample(t, bktName, objName)
|
|
req.Header.Set(api.ContentEncoding, "gzip")
|
|
hc.Handler().PutObjectHandler(w, req)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
resp = headObjectBase(hc, bktName, objName, emptyVersion)
|
|
require.Equal(t, "gzip", resp.Header().Get(api.ContentEncoding))
|
|
}
|
|
|
|
// getChunkedRequestAWSExample implements request example from
|
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
|
|
func getChunkedRequestAWSExample(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
|
|
chunk := make([]byte, 65*1024)
|
|
for i := range chunk {
|
|
chunk[i] = 'a'
|
|
}
|
|
chunk1 := chunk[:64*1024]
|
|
chunk2 := chunk[64*1024:]
|
|
|
|
AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
|
|
|
reqBody := bytes.NewBufferString("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n")
|
|
_, err := reqBody.Write(chunk1)
|
|
require.NoError(t, err)
|
|
_, err = reqBody.WriteString("\r\n400;chunk-signature=0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497\r\n")
|
|
require.NoError(t, err)
|
|
_, err = reqBody.Write(chunk2)
|
|
require.NoError(t, err)
|
|
_, err = reqBody.WriteString("\r\n0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n\r\n")
|
|
require.NoError(t, err)
|
|
|
|
req, err := http.NewRequest("PUT", "https://s3.amazonaws.com/"+bktName+"/"+objName, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("content-encoding", api.AwsChunked)
|
|
req.Header.Set("content-length", strconv.Itoa(awsChunkedRequestExampleContentLength))
|
|
req.Header.Set("x-amz-content-sha256", api.StreamingContentSHA256)
|
|
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
|
|
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
|
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9")
|
|
|
|
signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z")
|
|
require.NoError(t, err)
|
|
|
|
req.Body = io.NopCloser(reqBody)
|
|
|
|
w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
|
return w, req, chunk
|
|
}
|
|
|
|
type customNopCloser struct {
|
|
*bytes.Buffer
|
|
}
|
|
|
|
func (c *customNopCloser) Close() error {
|
|
return nil
|
|
}
|
|
|
|
// getChunkedRequestAWSExampleTrailing implements request example from
|
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html
|
|
func getChunkedRequestAWSExampleTrailing(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
|
|
chunk := make([]byte, 65*1024)
|
|
for i := range chunk {
|
|
chunk[i] = 'a'
|
|
}
|
|
chunk1 := chunk[:64*1024]
|
|
chunk2 := chunk[64*1024:]
|
|
|
|
AWSSecretAccessKey := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
|
|
|
reqBody := bytes.NewBufferString("10000;chunk-signature=b474d8862b1487a5145d686f57f013e54db672cee1c953b3010fb58501ef5aa2\r\n")
|
|
_, err := reqBody.Write(chunk1)
|
|
require.NoError(t, err)
|
|
_, err = reqBody.WriteString("\r\n400;chunk-signature=1c1344b170168f8e65b41376b44b20fe354e373826ccbbe2c1d40a8cae51e5c7\r\n")
|
|
require.NoError(t, err)
|
|
_, err = reqBody.Write(chunk2)
|
|
require.NoError(t, err)
|
|
_, err = reqBody.WriteString("\r\n0;chunk-signature=2ca2aba2005185cf7159c6277faf83795951dd77a3a99e6e65d5c9f85863f992\r\n")
|
|
require.NoError(t, err)
|
|
_, err = reqBody.WriteString("x-amz-checksum-crc32c:sOO8/Q==\n")
|
|
require.NoError(t, err)
|
|
|
|
// original signature is 63bddb248ad2590c92712055f51b8e78ab024eead08276b24f010b0efd74843f,
|
|
// but we use d81f82fc3505edab99d459891051a732e8730629a2e4a59689829ca17fe2e435
|
|
// because original signature is incorrect
|
|
// it was calculated using the`AWS4-HMAC-SHA256-PAYLOAD` constant in canonical string instead of
|
|
// `AWS4-HMAC-SHA256-TRAILER` that actually must be used by spec
|
|
// (java sdk use correct `AWS4-HMAC-SHA256-TRAILER` string).
|
|
_, err = reqBody.WriteString("x-amz-trailer-signature:d81f82fc3505edab99d459891051a732e8730629a2e4a59689829ca17fe2e435")
|
|
require.NoError(t, err)
|
|
|
|
req, err := http.NewRequest("PUT", "https://s3.amazonaws.com/"+bktName+"/"+objName, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("content-encoding", api.AwsChunked)
|
|
req.Header.Set("content-length", strconv.Itoa(awsChunkedRequestExampleContentLength))
|
|
req.Header.Set("x-amz-content-sha256", api.StreamingContentSHA256Trailer)
|
|
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
|
|
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
|
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32c")
|
|
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e")
|
|
|
|
signTime, err := time.Parse("20060102T150405Z", "20130524T000000Z")
|
|
require.NoError(t, err)
|
|
|
|
req.Body = &customNopCloser{Buffer: reqBody}
|
|
|
|
w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
|
return w, req, chunk
|
|
}
|
|
|
|
func getChunkedRequestUnsignedTrailing(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
|
|
chunk := make([]byte, 65*1024)
|
|
for i := range chunk {
|
|
chunk[i] = 'a'
|
|
}
|
|
|
|
AWSAccessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6"
|
|
AWSSecretAccessKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e"
|
|
|
|
awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey}
|
|
signer := v4.NewSigner()
|
|
|
|
reqBody := bytes.NewBufferString("10400\r\n")
|
|
_, err := reqBody.Write(chunk)
|
|
require.NoError(t, err)
|
|
_, err = reqBody.WriteString("\r\n0\r\n")
|
|
require.NoError(t, err)
|
|
_, err = reqBody.WriteString("\r\nx-amz-checksum-crc64nvme:pRf+emrnL+A=\r\n\r\n")
|
|
require.NoError(t, err)
|
|
|
|
req, err := http.NewRequest("PUT", "https://localhost:8184/"+bktName+"/"+objName, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("x-amz-sdk-checksum-algorithm", "CRC64NVME")
|
|
req.Header.Set("content-encoding", api.AwsChunked)
|
|
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc64nvme")
|
|
req.Header.Set("x-amz-content-sha256", api.StreamingUnsignedPayloadTrailer)
|
|
req.Header.Set("x-amz-decoded-content-length", strconv.Itoa(awsChunkedRequestExampleDecodedContentLength))
|
|
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
|
|
|
signTime, err := time.Parse("20060102T150405Z", "20250131T140527Z")
|
|
require.NoError(t, err)
|
|
|
|
err = signer.SignHTTP(ctx, awsCreds, req, api.StreamingContentSHA256Trailer, "s3", "ru", signTime)
|
|
require.NoError(t, err)
|
|
|
|
req.Body = io.NopCloser(reqBody)
|
|
|
|
w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
|
return w, req, chunk
|
|
}
|
|
|
|
func getChunkedRequestUnsignedTrailingSmall(ctx context.Context, t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request, []byte) {
|
|
AWSAccessKeyID := "9uEm8zMrGWsEDWiPCnVuQLKTiGtCEXpYXt8eBG7agupw0JDySJZMFuej7PTcPzRqBUyPtFowNu1RtvHULU8XHjie6"
|
|
AWSSecretAccessKey := "9f546428957ed7e189b7be928906ce7d1d9cb3042dd4d2d5194e28ce8c4c3b8e"
|
|
|
|
awsCreds := aws.Credentials{AccessKeyID: AWSAccessKeyID, SecretAccessKey: AWSSecretAccessKey}
|
|
signer := v4.NewSigner()
|
|
|
|
chunk := "tmp2\n"
|
|
|
|
reqBody := bytes.NewBufferString("5\r\n")
|
|
_, err := reqBody.WriteString(chunk)
|
|
require.NoError(t, err)
|
|
_, err = reqBody.WriteString("\r\n0\r\n")
|
|
require.NoError(t, err)
|
|
_, err = reqBody.WriteString("x-amz-checksum-crc64nvme:q1EYl4rI0TU=\r\n\r\n")
|
|
require.NoError(t, err)
|
|
|
|
req, err := http.NewRequest("PUT", "https://localhost:8184/"+bktName+"/"+objName, nil)
|
|
require.NoError(t, err)
|
|
req.Header.Set("x-amz-sdk-checksum-algorithm", "CRC64NVME")
|
|
req.Header.Set("content-encoding", api.AwsChunked)
|
|
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc64nvme")
|
|
req.Header.Set("x-amz-content-sha256", api.StreamingUnsignedPayloadTrailer)
|
|
req.Header.Set("x-amz-decoded-content-length", "5")
|
|
req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
|
|
|
|
signTime, err := time.Parse("20060102T150405Z", "20250203T063745Z")
|
|
require.NoError(t, err)
|
|
|
|
err = signer.SignHTTP(ctx, awsCreds, req, api.StreamingContentSHA256Trailer, "s3", "ru", signTime)
|
|
require.NoError(t, err)
|
|
|
|
req.Body = io.NopCloser(reqBody)
|
|
|
|
w, req := prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
|
return w, req, []byte(chunk)
|
|
}
|
|
|
|
func getChunkedRequestBase(t *testing.T, bktName, objName string, chunks [][]byte, shaType string, signTime time.Time, extraHeaders ...[2]string) (*httptest.ResponseRecorder, *http.Request) {
|
|
creds := aws.Credentials{
|
|
AccessKeyID: "48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh",
|
|
SecretAccessKey: "09260955b4eb0279dc017ba20a1ddac909cbd226c86cbb2d868e55534c8e64b0",
|
|
}
|
|
region := "us-east-1"
|
|
service := "s3"
|
|
|
|
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, nil)
|
|
require.NoError(t, err)
|
|
|
|
payloadLength := 0
|
|
for _, chunk := range chunks {
|
|
payloadLength += len(chunk)
|
|
}
|
|
|
|
for _, kv := range extraHeaders {
|
|
req.Header.Set(kv[0], kv[1])
|
|
}
|
|
|
|
req.Header.Set(api.ContentEncoding, api.AwsChunked)
|
|
req.Header.Set(api.AmzDecodedContentLength, strconv.Itoa(payloadLength))
|
|
req.Header.Set(api.AmzDate, signTime.Format("20060102T150405Z"))
|
|
req.Header.Set(api.AmzContentSha256, shaType)
|
|
if shaType == api.StreamingContentSHA256Trailer {
|
|
req.Header.Set(api.AmzTrailer, "x-amz-checksum-crc32")
|
|
}
|
|
|
|
signer := v4.NewSigner()
|
|
err = signer.SignHTTP(req.Context(), creds, req, shaType, service, region, signTime)
|
|
require.NoError(t, err)
|
|
|
|
seedSignature := strings.Split(req.Header.Get(api.Authorization), "Signature=")[1]
|
|
seed, err := hex.DecodeString(seedSignature)
|
|
require.NoError(t, err)
|
|
|
|
var reqBody bytes.Buffer
|
|
|
|
hash := crc32.NewIEEE()
|
|
|
|
newStreamSigner := v4.NewStreamSigner(creds, service, region, seed)
|
|
for _, chunk := range chunks {
|
|
_, err = hash.Write(chunk)
|
|
require.NoError(t, err)
|
|
|
|
signature, err := newStreamSigner.GetSignature(req.Context(), nil, chunk, signTime)
|
|
require.NoError(t, err)
|
|
reqBody.WriteString(fmt.Sprintf("%x;chunk-signature=%x\r\n", len(chunk), signature))
|
|
reqBody.Write(chunk)
|
|
reqBody.WriteString("\r\n")
|
|
}
|
|
signature, err := newStreamSigner.GetSignature(req.Context(), nil, nil, signTime)
|
|
require.NoError(t, err)
|
|
reqBody.WriteString(fmt.Sprintf("0;chunk-signature=%x\r\n", signature))
|
|
|
|
if shaType == api.StreamingContentSHA256Trailer {
|
|
crc32Res := hash.Sum(nil)
|
|
checksumStr := "x-amz-checksum-crc32:" + base64.StdEncoding.EncodeToString(crc32Res)
|
|
reqBody.WriteString(fmt.Sprintf("%s\r\n", checksumStr))
|
|
trailerSignature, err := newStreamSigner.GetTrailerSignature([]byte(checksumStr+"\n"), signTime)
|
|
require.NoError(t, err)
|
|
reqBody.WriteString(fmt.Sprintf("x-amz-trailer-signature:%x\r\n", trailerSignature))
|
|
}
|
|
reqBody.WriteString("\r\n")
|
|
|
|
req.Body = io.NopCloser(&reqBody)
|
|
|
|
return prepareReqMiddlewares(req, signTime, creds.SecretAccessKey)
|
|
}
|
|
|
|
func prepareReqMiddlewares(req *http.Request, signTime time.Time, secretAccessKey string) (*httptest.ResponseRecorder, *http.Request) {
|
|
authHeader := req.Header.Get(api.Authorization)
|
|
var parsed map[string]string
|
|
var region string
|
|
if strings.HasPrefix(authHeader, auth.SignaturePreambleSigV4) {
|
|
parsed = auth.NewRegexpMatcher(auth.AuthorizationFieldRegexp).GetSubmatches(authHeader)
|
|
region = parsed["region"]
|
|
} else {
|
|
parsed = auth.NewRegexpMatcher(auth.AuthorizationFieldV4aRegexp).GetSubmatches(authHeader)
|
|
region = req.Header.Get("X-Amz-Region-Set")
|
|
}
|
|
|
|
bktObj := strings.Split(req.URL.Path, "/")
|
|
|
|
box := &middleware.Box{
|
|
ClientTime: signTime,
|
|
AuthHeaders: &middleware.AuthHeader{
|
|
AccessKeyID: parsed["access_key_id"],
|
|
SignatureV4: parsed["v4_signature"],
|
|
Region: region,
|
|
},
|
|
AccessBox: &accessbox.Box{Gate: &accessbox.GateData{SecretKey: secretAccessKey}},
|
|
}
|
|
|
|
w := httptest.NewRecorder()
|
|
reqInfo := middleware.NewReqInfo(w, req, middleware.ObjectRequest{Bucket: bktObj[1], Object: bktObj[2]}, "")
|
|
req = req.WithContext(middleware.SetReqInfo(req.Context(), reqInfo))
|
|
req = req.WithContext(middleware.SetBox(req.Context(), box))
|
|
|
|
return w, req
|
|
}
|
|
|
|
func getEmptyChunkedRequestUnsigned(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) {
|
|
AWSSecretAccessKey := "f1a0d650b650149f1a83140418e88a3c5572a0103e912e326492a91c19c4488a"
|
|
|
|
reqBody := bytes.NewBufferString("0\r\nx-amz-checksum-crc64nvme:AAAAAAAAAAA=\r\n\r\n")
|
|
|
|
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, reqBody)
|
|
require.NoError(t, err)
|
|
req.Header.Set(api.Authorization, "AWS4-HMAC-SHA256 Credential=3jNrmDtHtuj1uLcixaSMA4KNUhNYhv1EpUNdFnbTXgUP071pGdSZfHSLtoC8gzjF5HoD6sC3Scq33t1WvvEvjmPnt/20250213/ru/s3/aws4_request, SignedHeaders=content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-sdk-checksum-algorithm;x-amz-trailer, Signature=1231b012c0ac313770c5a95ccf77b95b6c9b1c3760d6aa24cb8309801d56eb4a")
|
|
req.Header.Set(api.ContentEncoding, api.AwsChunked)
|
|
req.Header.Set(api.AmzDate, "20250213T124858Z")
|
|
req.Header.Set(api.AmzContentSha256, api.StreamingUnsignedPayloadTrailer)
|
|
req.Header.Set(api.AmzDecodedContentLength, "0")
|
|
req.Header.Set("X-Amz-Trailer", "x-amz-checksum-crc64nvme")
|
|
req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC64NVME")
|
|
|
|
signTime, err := time.Parse("20060102T150405Z", req.Header.Get(api.AmzDate))
|
|
require.NoError(t, err)
|
|
|
|
return prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
|
}
|
|
|
|
func getEmptyChunkedRequestSigv4aWithTrailers(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) {
|
|
AWSSecretAccessKey := "f1a0d650b650149f1a83140418e88a3c5572a0103e912e326492a91c19c4488a"
|
|
|
|
body := "0;chunk-signature=3046022100ab9229a80d70f4d004768992881821a441a4ad4102e18de567e68216659bf497022100ec47a7a445351683557eedf893e6ed250c97af4b0415814671770b83766d69be\r\n" +
|
|
"x-amz-checksum-crc32:AAAAAA==\r\n" +
|
|
"x-amz-trailer-signature:3046022100a0a66c1adcee8d99460b4631b23c95fbad9eb4e6c56f1afb9e255715ba141169022100b2cfc8adc8036eb985f1ab0e770b575284c5fc8ca75c226558d3142cbaab83ce\r\n\r\n"
|
|
|
|
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, bytes.NewBufferString(body))
|
|
require.NoError(t, err)
|
|
req.Header.Set(api.Authorization, "AWS4-ECDSA-P256-SHA256 Credential=3jNrmDtHtuj1uLcixaSMA4KNUhNYhv1EpUNdFnbTXgUP071pGdSZfHSLtoC8gzjF5HoD6sC3Scq33t1WvvEvjmPnt/20250213/s3/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-region-set;x-amz-sdk-checksum-algorithm;x-amz-trailer, Signature=304402202e1f1efcc56c588d9a94a3d8f20368686df8bfd5e8aad01fc4eff569ff38f1800220215198e3f1ba785492fe6703c4722872909ce8a09e8c9a13da90a9230c7a24b7")
|
|
req.Header.Set("Amz-Sdk-Invocation-Id", "d42dc16d-7899-55fb-5b72-a654bd482f4f")
|
|
req.Header.Set("Amz-Sdk-Request", "attempt=1; max=2")
|
|
req.Header.Set(api.ContentEncoding, api.AwsChunked)
|
|
req.Header.Set(api.AmzDate, "20250213T132401Z")
|
|
req.Header.Set(api.AmzContentSha256, api.StreamingContentV4aSHA256Trailer)
|
|
req.Header.Set(api.AmzDecodedContentLength, "0")
|
|
req.Header.Set(api.ContentLength, "367")
|
|
req.Header.Set(api.ContentType, "text/plain: charset=UTF-8")
|
|
req.Header.Set("X-Amz-Region-Set", "us-east-1")
|
|
req.Header.Set("X-Amz-Trailer", "x-amz-checksum-crc32")
|
|
req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC32")
|
|
|
|
signTime, err := time.Parse("20060102T150405Z", req.Header.Get(api.AmzDate))
|
|
require.NoError(t, err)
|
|
|
|
return prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
|
}
|
|
|
|
func getEmptyChunkedRequestSigv4a(t *testing.T, bktName, objName string) (*httptest.ResponseRecorder, *http.Request) {
|
|
AWSSecretAccessKey := "f1a0d650b650149f1a83140418e88a3c5572a0103e912e326492a91c19c4488a"
|
|
|
|
body := "0;chunk-signature=304502203f7c598a2e9a6673bf1ca30f5f6bebd0d76a4e9d3c16531448e96c2cda22d16a0221009e7ed578da0a9781366f1461a1484e64f15707f26d4310e59514db6ff9f7e0f1**\r\n\r\n"
|
|
|
|
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, bytes.NewBufferString(body))
|
|
require.NoError(t, err)
|
|
req.Header.Set(api.Authorization, "AWS4-ECDSA-P256-SHA256 Credential=3jNrmDtHtuj1uLcixaSMA4KNUhNYhv1EpUNdFnbTXgUP071pGdSZfHSLtoC8gzjF5HoD6sC3Scq33t1WvvEvjmPnt/20250213/s3/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-region-set, Signature=3046022100dc589ea513448b996809db4b314a0b8a4a775c1165c6203c7104b2f1aae1243c0221009bf3a256e7c33415eaad20c1dbfb4e14cb00b362758bc4d2aaf94ca96a5f13f9")
|
|
req.Header.Set("Amz-Sdk-Invocation-Id", "f0814a40-0d74-066f-d01f-ed14f28ebfa4")
|
|
req.Header.Set("Amz-Sdk-Request", "attempt=1; max=2")
|
|
req.Header.Set(api.ContentEncoding, api.AwsChunked)
|
|
req.Header.Set(api.AmzDate, "20250213T135717Z")
|
|
req.Header.Set(api.AmzContentSha256, api.StreamingContentV4aSHA256)
|
|
req.Header.Set(api.AmzDecodedContentLength, "0")
|
|
req.Header.Set(api.ContentLength, "166")
|
|
req.Header.Set(api.ContentType, "text/plain: charset=UTF-8")
|
|
req.Header.Set("X-Amz-Region-Set", "use-east-1")
|
|
|
|
signTime, err := time.Parse("20060102T150405Z", req.Header.Get(api.AmzDate))
|
|
require.NoError(t, err)
|
|
|
|
return prepareReqMiddlewares(req, signTime, AWSSecretAccessKey)
|
|
}
|
|
|
|
func TestCreateBucket(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
bktName := "bkt-name"
|
|
|
|
info := createBucket(hc, bktName)
|
|
createBucketAssertS3Error(hc, bktName, info.Box, apierr.ErrBucketAlreadyOwnedByYou)
|
|
|
|
box2, _ := createAccessBox(t)
|
|
createBucketAssertS3Error(hc, bktName, box2, apierr.ErrBucketAlreadyExists)
|
|
}
|
|
|
|
func TestCreateBucketWithoutPermissions(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
bktName := "bkt-name"
|
|
|
|
hc.h.ape.(*apeMock).err = errors.New("no permissions")
|
|
|
|
box, _ := createAccessBox(t)
|
|
createBucketAssertS3Error(hc, bktName, box, apierr.ErrInternalError)
|
|
|
|
_, err := hc.tp.ContainerID(bktName)
|
|
require.Errorf(t, err, "container exists after failed creation, but shouldn't")
|
|
}
|
|
|
|
func TestCreateNamespacedBucket(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
bktName := "bkt-name"
|
|
namespace := "yabloko"
|
|
|
|
box, _ := createAccessBox(t)
|
|
w, r := prepareTestRequest(hc, bktName, "", nil)
|
|
ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
|
|
reqInfo := middleware.GetReqInfo(ctx)
|
|
reqInfo.Namespace = namespace
|
|
r = r.WithContext(middleware.SetReqInfo(ctx, reqInfo))
|
|
hc.Handler().CreateBucketHandler(w, r)
|
|
assertStatus(t, w, http.StatusOK)
|
|
|
|
bktInfo, err := hc.Layer().GetBucketInfo(middleware.SetReqInfo(hc.Context(), reqInfo), bktName)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, namespace+".ns", bktInfo.Zone)
|
|
}
|
|
|
|
func TestPutObjectClientCut(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
bktName, objName1, objName2 := "bkt-name", "obj-name1", "obj-name2"
|
|
createTestBucket(hc, bktName)
|
|
|
|
putObject(hc, bktName, objName1)
|
|
obj1 := getObjectFromLayer(hc, objName1)[0]
|
|
require.Empty(t, getObjectAttribute(obj1, "s3-client-cut"))
|
|
|
|
hc.layerFeatures.SetClientCut(true)
|
|
putObject(hc, bktName, objName2)
|
|
obj2 := getObjectFromLayer(hc, objName2)[0]
|
|
require.Equal(t, "true", getObjectAttribute(obj2, "s3-client-cut"))
|
|
}
|
|
|
|
func getObjectFromLayer(hc *handlerContext, objName string) []*object.Object {
|
|
var res []*object.Object
|
|
for _, o := range hc.tp.Objects() {
|
|
if objName == getObjectAttribute(o, object.AttributeFilePath) {
|
|
res = append(res, o)
|
|
}
|
|
}
|
|
return res
|
|
}
|
|
|
|
func getObjectAttribute(obj *object.Object, attrName string) string {
|
|
for _, attr := range obj.Attributes() {
|
|
if attr.Key() == attrName {
|
|
return attr.Value()
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func TestPutObjectWithContentLanguage(t *testing.T) {
|
|
tc := prepareHandlerContext(t)
|
|
|
|
expectedContentLanguage := "en"
|
|
bktName, objName := "bucket-1", "object-1"
|
|
createTestBucket(tc, bktName)
|
|
|
|
w, r := prepareTestRequest(tc, bktName, objName, nil)
|
|
r.Header.Set(api.ContentLanguage, expectedContentLanguage)
|
|
tc.Handler().PutObjectHandler(w, r)
|
|
|
|
tc.Handler().HeadObjectHandler(w, r)
|
|
require.Equal(t, expectedContentLanguage, w.Header().Get(api.ContentLanguage))
|
|
}
|
|
|
|
func TestFormEncryptionParamsBase(t *testing.T) {
|
|
userSecret := "test1customer2secret3with32char4"
|
|
expectedEncKey := []byte(userSecret)
|
|
emptyEncKey := []byte(nil)
|
|
|
|
validAlgo := "AES256"
|
|
validKey := "dGVzdDFjdXN0b21lcjJzZWNyZXQzd2l0aDMyY2hhcjQ="
|
|
validMD5 := "zcQmPqFhtJaxkOIg5tXm9g=="
|
|
|
|
invalidAlgo := "TTT111"
|
|
invalidKeyBase64 := "dGVzdDFjdXN0b21lcjJzZWNyZXQzd2l0aDMyY2hhcjQ"
|
|
invalidKeySize := "dGVzdDFjdXN0b21lcjJzZWNyZXQzd2l0aA=="
|
|
invalidMD5Base64 := "zcQmPqFhtJaxkOIg5tXm9g"
|
|
invalidMD5 := "zcQmPqPhtJaxkOIg5tXm9g=="
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
algo string
|
|
key string
|
|
md5 string
|
|
tlsTermination string
|
|
reqWithoutTLS bool
|
|
reqWithoutSSE bool
|
|
isCopySource bool
|
|
err error
|
|
}{
|
|
{
|
|
name: "valid requst copy source",
|
|
algo: validAlgo,
|
|
key: validKey,
|
|
md5: validMD5,
|
|
isCopySource: true,
|
|
},
|
|
{
|
|
name: "valid request with TLS",
|
|
algo: validAlgo,
|
|
key: validKey,
|
|
md5: validMD5,
|
|
},
|
|
{
|
|
name: "valid request without TLS and valid termination header",
|
|
algo: validAlgo,
|
|
key: validKey,
|
|
md5: validMD5,
|
|
tlsTermination: "true",
|
|
reqWithoutTLS: true,
|
|
},
|
|
{
|
|
name: "request without tls and termination header",
|
|
algo: validAlgo,
|
|
key: validKey,
|
|
md5: validMD5,
|
|
reqWithoutTLS: true,
|
|
err: apierr.GetAPIError(apierr.ErrInsecureSSECustomerRequest),
|
|
},
|
|
{
|
|
name: "request without tls and invalid header",
|
|
algo: validAlgo,
|
|
key: validKey,
|
|
md5: validMD5,
|
|
tlsTermination: "invalid",
|
|
reqWithoutTLS: true,
|
|
err: apierr.GetAPIError(apierr.ErrInsecureSSECustomerRequest),
|
|
},
|
|
{
|
|
name: "missing SSE customer algorithm",
|
|
key: validKey,
|
|
md5: validMD5,
|
|
err: apierr.GetAPIError(apierr.ErrMissingSSECustomerAlgorithm),
|
|
},
|
|
{
|
|
name: "missing SSE customer key",
|
|
algo: validAlgo,
|
|
md5: validMD5,
|
|
err: apierr.GetAPIError(apierr.ErrMissingSSECustomerKey),
|
|
},
|
|
{
|
|
name: "invalid encryption algorithm",
|
|
algo: invalidAlgo,
|
|
key: validKey,
|
|
md5: validMD5,
|
|
err: apierr.GetAPIError(apierr.ErrInvalidEncryptionAlgorithm),
|
|
},
|
|
{
|
|
name: "invalid base64 SSE customer key",
|
|
algo: validAlgo,
|
|
key: invalidKeyBase64,
|
|
md5: validMD5,
|
|
err: apierr.GetAPIError(apierr.ErrInvalidSSECustomerKey),
|
|
},
|
|
{
|
|
name: "invalid base64 SSE customer parameters",
|
|
algo: validAlgo,
|
|
key: invalidKeyBase64,
|
|
md5: validMD5,
|
|
isCopySource: true,
|
|
err: apierr.GetAPIError(apierr.ErrInvalidSSECustomerParameters),
|
|
},
|
|
{
|
|
name: "invalid size of custom key",
|
|
algo: validAlgo,
|
|
key: invalidKeySize,
|
|
md5: validMD5,
|
|
err: apierr.GetAPIError(apierr.ErrInvalidSSECustomerKey),
|
|
},
|
|
{
|
|
name: "invalid size of custom key - copy source",
|
|
algo: validAlgo,
|
|
key: invalidKeySize,
|
|
md5: validMD5,
|
|
isCopySource: true,
|
|
err: apierr.GetAPIError(apierr.ErrInvalidSSECustomerParameters),
|
|
},
|
|
{
|
|
name: "invalid base64 key md5 of customer",
|
|
algo: validAlgo,
|
|
key: validKey,
|
|
md5: invalidMD5Base64,
|
|
err: apierr.GetAPIError(apierr.ErrSSECustomerKeyMD5Mismatch),
|
|
},
|
|
{
|
|
name: "invalid md5 sum key of customer",
|
|
algo: validAlgo,
|
|
key: validKey,
|
|
md5: invalidMD5,
|
|
err: apierr.GetAPIError(apierr.ErrSSECustomerKeyMD5Mismatch),
|
|
},
|
|
{
|
|
name: "request without sse",
|
|
reqWithoutSSE: true,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
r := prepareRequestForEncryption(hc, tc.algo, tc.key, tc.md5, tc.tlsTermination, tc.reqWithoutTLS, tc.reqWithoutSSE, tc.isCopySource)
|
|
|
|
enc, err := hc.h.formEncryptionParamsBase(r, tc.isCopySource)
|
|
if tc.err != nil {
|
|
require.ErrorIs(t, tc.err, err)
|
|
return
|
|
}
|
|
|
|
require.NoError(t, err)
|
|
|
|
if tc.reqWithoutSSE {
|
|
require.Equal(t, emptyEncKey, enc.Key())
|
|
} else {
|
|
require.Equal(t, expectedEncKey, enc.Key())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCheckContentLength(t *testing.T) {
|
|
contentLength := "content-length-range"
|
|
notFallError := "length of the content did not fall within the range specified in the condition"
|
|
parseError := "invalid condition"
|
|
for _, tc := range []struct {
|
|
name string
|
|
matching string
|
|
key string
|
|
value string
|
|
size uint64
|
|
errMsg string
|
|
emptyPolicy bool
|
|
}{
|
|
{
|
|
name: "valid",
|
|
matching: contentLength,
|
|
key: "0",
|
|
value: "1000",
|
|
size: 50,
|
|
},
|
|
{
|
|
name: "valid lower limit",
|
|
matching: contentLength,
|
|
key: "5",
|
|
value: "100",
|
|
size: 5,
|
|
},
|
|
{
|
|
name: "valid upper limit",
|
|
matching: contentLength,
|
|
key: "5",
|
|
value: "100",
|
|
size: 100,
|
|
},
|
|
{
|
|
name: "invalid size value (too small)",
|
|
matching: contentLength,
|
|
key: "5",
|
|
value: "100",
|
|
size: 2,
|
|
errMsg: notFallError,
|
|
},
|
|
{
|
|
name: "invalid size value (to high)",
|
|
matching: contentLength,
|
|
key: "5",
|
|
value: "100",
|
|
size: 200,
|
|
errMsg: notFallError,
|
|
},
|
|
{
|
|
name: "no matching",
|
|
},
|
|
{
|
|
name: "invalid key type",
|
|
matching: contentLength,
|
|
key: "invalid",
|
|
value: "100",
|
|
size: 10,
|
|
errMsg: parseError,
|
|
},
|
|
{
|
|
name: "invalid value type",
|
|
matching: contentLength,
|
|
key: "5",
|
|
value: "invalid",
|
|
size: 10,
|
|
errMsg: parseError,
|
|
},
|
|
{
|
|
name: "empty policy",
|
|
emptyPolicy: true,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
policy := &postPolicy{
|
|
Conditions: []*policyCondition{
|
|
{
|
|
Matching: tc.matching,
|
|
Key: tc.key,
|
|
Value: tc.value,
|
|
},
|
|
},
|
|
empty: tc.emptyPolicy,
|
|
}
|
|
|
|
err := policy.CheckContentLength(tc.size)
|
|
if tc.errMsg != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), tc.errMsg)
|
|
return
|
|
}
|
|
|
|
require.NoError(t, err)
|
|
})
|
|
}
|
|
}
|
|
|
|
func prepareRequestForEncryption(hc *handlerContext, algo, key, md5, tlsTermination string, reqWithoutTLS, reqWithoutSSE, isCopySource bool) *http.Request {
|
|
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
|
|
|
if !reqWithoutTLS {
|
|
r.TLS = &tls.ConnectionState{}
|
|
}
|
|
|
|
if !reqWithoutSSE {
|
|
if isCopySource {
|
|
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerAlgorithm, algo)
|
|
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKey, key)
|
|
r.Header.Set(api.AmzCopySourceServerSideEncryptionCustomerKeyMD5, md5)
|
|
} else {
|
|
r.Header.Set(api.AmzServerSideEncryptionCustomerAlgorithm, algo)
|
|
r.Header.Set(api.AmzServerSideEncryptionCustomerKey, key)
|
|
r.Header.Set(api.AmzServerSideEncryptionCustomerKeyMD5, md5)
|
|
}
|
|
}
|
|
|
|
customHeader := "X-Frostfs-TLS-Termination"
|
|
if tlsTermination != "" {
|
|
hc.config.tlsTerminationHeader = customHeader
|
|
r.Header.Set(customHeader, tlsTermination)
|
|
}
|
|
|
|
return r
|
|
}
|
|
|
|
func postObjectBase(hc *handlerContext, ns, bktName, key, filename, content string) *httptest.ResponseRecorder {
|
|
policy := "eyJleHBpcmF0aW9uIjogIjIwMjUtMTItMDFUMTI6MDA6MDAuMDAwWiIsImNvbmRpdGlvbnMiOiBbCiBbInN0YXJ0cy13aXRoIiwgIiR4LWFtei1jcmVkZW50aWFsIiwgIiJdLAogWyJzdGFydHMtd2l0aCIsICIkeC1hbXotZGF0ZSIsICIiXSwKIFsic3RhcnRzLXdpdGgiLCAiJGtleSIsICIiXQpdfQ=="
|
|
|
|
timeToSign := time.Now()
|
|
timeToSignStr := timeToSign.Format("20060102T150405Z")
|
|
region := "default"
|
|
service := "s3"
|
|
|
|
accessKeyID := "5jizSbYu8hX345aqCKDgRWKCJYHxnzxRS8e6SUYHZ8Fw0HiRkf3KbJAWBn5mRzmiyHQ3UHADGyzVXLusn1BrmAfLn"
|
|
secretKey := "abf066d77c6744cd956a123a0b9612df587f5c14d3350ecb01b363f182dd7279"
|
|
|
|
creds := getCredsStr(accessKeyID, timeToSignStr, region, service)
|
|
sign := auth.SignStr(secretKey, service, region, timeToSign, policy)
|
|
|
|
body, contentType, err := getMultipartFormBody(policy, creds, timeToSignStr, sign, key, filename, content)
|
|
require.NoError(hc.t, err)
|
|
|
|
w, r := prepareTestPostRequest(hc, bktName, body)
|
|
r.Header.Set(auth.ContentTypeHdr, contentType)
|
|
r.Header.Set("X-Frostfs-Namespace", ns)
|
|
|
|
err = r.ParseMultipartForm(50 * 1024 * 1024)
|
|
require.NoError(hc.t, err)
|
|
|
|
hc.Handler().PostObject(w, r)
|
|
return w
|
|
}
|
|
|
|
func getCredsStr(accessKeyID, timeToSign, region, service string) string {
|
|
return accessKeyID + "/" + timeToSign + "/" + region + "/" + service + "/aws4_request"
|
|
}
|
|
|
|
func getMultipartFormBody(policy, creds, date, sign, key, filename, content string) (io.Reader, string, error) {
|
|
body := &bytes.Buffer{}
|
|
writer := multipart.NewWriter(body)
|
|
defer writer.Close()
|
|
|
|
if err := writer.WriteField("policy", policy); err != nil {
|
|
return nil, "", err
|
|
}
|
|
|
|
if err := writer.WriteField("key", key); err != nil {
|
|
return nil, "", err
|
|
}
|
|
if err := writer.WriteField(strings.ToLower(auth.AmzCredential), creds); err != nil {
|
|
return nil, "", err
|
|
}
|
|
if err := writer.WriteField(strings.ToLower(auth.AmzDate), date); err != nil {
|
|
return nil, "", err
|
|
}
|
|
if err := writer.WriteField(strings.ToLower(auth.AmzSignature), sign); err != nil {
|
|
return nil, "", err
|
|
}
|
|
|
|
file, err := writer.CreateFormFile("file", filename)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
|
|
if len(content) < maxContentSizeForFormData {
|
|
if err = writer.WriteField("file", content); err != nil {
|
|
return nil, "", err
|
|
}
|
|
} else {
|
|
if _, err = file.Write([]byte(content)); err != nil {
|
|
return nil, "", err
|
|
}
|
|
}
|
|
|
|
return body, writer.FormDataContentType(), nil
|
|
}
|
|
|
|
func prepareTestPostRequest(hc *handlerContext, bktName string, payload io.Reader) (*httptest.ResponseRecorder, *http.Request) {
|
|
w := httptest.NewRecorder()
|
|
r := httptest.NewRequest(http.MethodPost, defaultURL+bktName, payload)
|
|
|
|
reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: bktName}, "")
|
|
r = r.WithContext(middleware.SetReqInfo(hc.Context(), reqInfo))
|
|
|
|
return w, r
|
|
}
|