forked from TrueCloudLab/frostfs-s3-gw
Denis Kirillov
5ee73fad6a
Despite the spec https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html#API_ListObjectVersions_ResponseElements says that "When the number of responses exceeds the value of MaxKeys, NextVersionIdMarker specifies the first object version not returned that satisfies the search criteria. Use this value for the version-id-marker request parameter in a subsequent request." the actual behavior of AWS S3 is returning NextVersionIdMarker as the last returned object version Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
330 lines
11 KiB
Go
330 lines
11 KiB
Go
package handler
|
|
|
|
import (
|
|
"net/http"
|
|
"net/url"
|
|
"sort"
|
|
"strconv"
|
|
"testing"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestParseContinuationToken(t *testing.T) {
|
|
var err error
|
|
|
|
t.Run("empty token", func(t *testing.T) {
|
|
var queryValues = map[string][]string{
|
|
"continuation-token": {""},
|
|
}
|
|
_, err = parseContinuationToken(queryValues)
|
|
require.Error(t, err)
|
|
})
|
|
|
|
t.Run("invalid not empty token", func(t *testing.T) {
|
|
var queryValues = map[string][]string{
|
|
"continuation-token": {"asd"},
|
|
}
|
|
_, err = parseContinuationToken(queryValues)
|
|
require.Error(t, err)
|
|
})
|
|
|
|
t.Run("valid token", func(t *testing.T) {
|
|
tokenStr := "75BTT5Z9o79XuKdUeGqvQbqDnxu6qWcR5EhxW8BXFf8t"
|
|
var queryValues = map[string][]string{
|
|
"continuation-token": {tokenStr},
|
|
}
|
|
token, err := parseContinuationToken(queryValues)
|
|
require.NoError(t, err)
|
|
require.Equal(t, tokenStr, token)
|
|
})
|
|
}
|
|
|
|
func TestListObjectNullVersions(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
bktName, objName := "bucket-versioning-enabled", "object"
|
|
createTestBucket(hc, bktName)
|
|
|
|
putObjectContent(hc, bktName, objName, "content")
|
|
putBucketVersioning(t, hc, bktName, true)
|
|
putObjectContent(hc, bktName, objName, "content2")
|
|
|
|
result := listVersions(t, hc, bktName)
|
|
|
|
require.Len(t, result.Version, 2)
|
|
require.Equal(t, data.UnversionedObjectVersionID, result.Version[1].VersionID)
|
|
}
|
|
|
|
func TestListObjectsPaging(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
bktName := "bucket-versioning-enabled"
|
|
createTestBucket(hc, bktName)
|
|
|
|
n := 10
|
|
|
|
var objects []string
|
|
for i := 0; i < n; i++ {
|
|
objects = append(objects, "objects"+strconv.Itoa(i))
|
|
putObjectContent(hc, bktName, objects[i], "content")
|
|
}
|
|
sort.Strings(objects)
|
|
|
|
result := &ListObjectsVersionsResponse{IsTruncated: true}
|
|
for result.IsTruncated {
|
|
result = listObjectsVersions(hc, bktName, "", "", result.NextKeyMarker, result.NextVersionIDMarker, n/3)
|
|
|
|
for i, version := range result.Version {
|
|
if objects[i] != version.Key {
|
|
t.Errorf("expected: '%s', got: '%s'", objects[i], version.Key)
|
|
}
|
|
}
|
|
objects = objects[len(result.Version):]
|
|
}
|
|
|
|
require.Empty(t, objects)
|
|
}
|
|
|
|
func TestS3CompatibilityBucketListV2BothContinuationTokenStartAfter(t *testing.T) {
|
|
tc := prepareHandlerContext(t)
|
|
|
|
bktName := "bucket-for-listing"
|
|
objects := []string{"bar", "baz", "foo", "quxx"}
|
|
bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])
|
|
|
|
for _, objName := range objects[1:] {
|
|
createTestObject(tc, bktInfo, objName)
|
|
}
|
|
|
|
listV2Response1 := listObjectsV2(tc, bktName, "", "", "bar", "", 1)
|
|
nextContinuationToken := listV2Response1.NextContinuationToken
|
|
require.Equal(t, "baz", listV2Response1.Contents[0].Key)
|
|
|
|
listV2Response2 := listObjectsV2(tc, bktName, "", "", "bar", nextContinuationToken, -1)
|
|
|
|
require.Equal(t, nextContinuationToken, listV2Response2.ContinuationToken)
|
|
require.Equal(t, "bar", listV2Response2.StartAfter)
|
|
require.False(t, listV2Response2.IsTruncated)
|
|
|
|
require.Equal(t, "foo", listV2Response2.Contents[0].Key)
|
|
require.Equal(t, "quxx", listV2Response2.Contents[1].Key)
|
|
}
|
|
|
|
func TestS3BucketListV2EncodingBasic(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
bktName := "bucket-for-listing-v1-encoding"
|
|
bktInfo := createTestBucket(hc, bktName)
|
|
|
|
objects := []string{"foo+1/bar", "foo/bar/xyzzy", "quux ab/thud", "asdf+b"}
|
|
for _, objName := range objects {
|
|
createTestObject(hc, bktInfo, objName)
|
|
}
|
|
|
|
query := make(url.Values)
|
|
query.Add("delimiter", "/")
|
|
query.Add("encoding-type", "url")
|
|
|
|
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
|
hc.Handler().ListObjectsV2Handler(w, r)
|
|
assertStatus(hc.t, w, http.StatusOK)
|
|
listV2Response := &ListObjectsV2Response{}
|
|
parseTestResponse(hc.t, w, listV2Response)
|
|
|
|
require.Equal(t, "/", listV2Response.Delimiter)
|
|
require.Len(t, listV2Response.Contents, 1)
|
|
require.Equal(t, "asdf%2Bb", listV2Response.Contents[0].Key)
|
|
require.Len(t, listV2Response.CommonPrefixes, 3)
|
|
require.Equal(t, "foo%2B1/", listV2Response.CommonPrefixes[0].Prefix)
|
|
require.Equal(t, "foo/", listV2Response.CommonPrefixes[1].Prefix)
|
|
require.Equal(t, "quux%20ab/", listV2Response.CommonPrefixes[2].Prefix)
|
|
}
|
|
|
|
func TestS3BucketListDelimiterBasic(t *testing.T) {
|
|
tc := prepareHandlerContext(t)
|
|
|
|
bktName := "bucket-for-listing"
|
|
objects := []string{"foo/bar", "foo/bar/xyzzy", "quux/thud", "asdf"}
|
|
bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])
|
|
|
|
for _, objName := range objects[1:] {
|
|
createTestObject(tc, bktInfo, objName)
|
|
}
|
|
|
|
listV1Response := listObjectsV1(tc, bktName, "", "/", "", -1)
|
|
require.Equal(t, "/", listV1Response.Delimiter)
|
|
require.Equal(t, "asdf", listV1Response.Contents[0].Key)
|
|
require.Len(t, listV1Response.CommonPrefixes, 2)
|
|
require.Equal(t, "foo/", listV1Response.CommonPrefixes[0].Prefix)
|
|
require.Equal(t, "quux/", listV1Response.CommonPrefixes[1].Prefix)
|
|
}
|
|
|
|
func TestS3BucketListV2DelimiterPercentage(t *testing.T) {
|
|
tc := prepareHandlerContext(t)
|
|
|
|
bktName := "bucket-for-listing"
|
|
objects := []string{"b%ar", "b%az", "c%ab", "foo"}
|
|
bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])
|
|
|
|
for _, objName := range objects[1:] {
|
|
createTestObject(tc, bktInfo, objName)
|
|
}
|
|
|
|
listV2Response := listObjectsV2(tc, bktName, "", "%", "", "", -1)
|
|
require.Equal(t, "%", listV2Response.Delimiter)
|
|
require.Len(t, listV2Response.Contents, 1)
|
|
require.Equal(t, "foo", listV2Response.Contents[0].Key)
|
|
require.Len(t, listV2Response.CommonPrefixes, 2)
|
|
require.Equal(t, "b%", listV2Response.CommonPrefixes[0].Prefix)
|
|
require.Equal(t, "c%", listV2Response.CommonPrefixes[1].Prefix)
|
|
}
|
|
|
|
func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
|
|
tc := prepareHandlerContext(t)
|
|
|
|
bktName := "bucket-for-listingv2"
|
|
objects := []string{"asdf", "boo/bar", "boo/baz/xyzzy", "cquux/thud", "cquux/bla"}
|
|
bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])
|
|
|
|
for _, objName := range objects[1:] {
|
|
createTestObject(tc, bktInfo, objName)
|
|
}
|
|
|
|
var empty []string
|
|
delim := "/"
|
|
prefix := ""
|
|
|
|
continuationToken := validateListV2(t, tc, bktName, prefix, delim, "", 1, true, false, []string{"asdf"}, empty)
|
|
continuationToken = validateListV2(t, tc, bktName, prefix, delim, continuationToken, 1, true, false, empty, []string{"boo/"})
|
|
validateListV2(t, tc, bktName, prefix, delim, continuationToken, 1, false, true, empty, []string{"cquux/"})
|
|
|
|
continuationToken = validateListV2(t, tc, bktName, prefix, delim, "", 2, true, false, []string{"asdf"}, []string{"boo/"})
|
|
validateListV2(t, tc, bktName, prefix, delim, continuationToken, 2, false, true, empty, []string{"cquux/"})
|
|
|
|
prefix = "boo/"
|
|
continuationToken = validateListV2(t, tc, bktName, prefix, delim, "", 1, true, false, []string{"boo/bar"}, empty)
|
|
validateListV2(t, tc, bktName, prefix, delim, continuationToken, 1, false, true, empty, []string{"boo/baz/"})
|
|
|
|
validateListV2(t, tc, bktName, prefix, delim, "", 2, false, true, []string{"boo/bar"}, []string{"boo/baz/"})
|
|
}
|
|
|
|
func TestMintVersioningListObjectVersionsVersionIDContinuation(t *testing.T) {
|
|
hc := prepareHandlerContext(t)
|
|
|
|
bktName, objName := "mint-bucket-for-listing-versions", "objName"
|
|
createTestBucket(hc, bktName)
|
|
putBucketVersioning(t, hc, bktName, true)
|
|
|
|
length := 10
|
|
objects := make([]string, length)
|
|
for i := 0; i < length; i++ {
|
|
objects[i] = objName
|
|
putObject(hc, bktName, objName)
|
|
}
|
|
|
|
maxKeys := 5
|
|
|
|
page1 := listObjectsVersions(hc, bktName, "", "", "", "", maxKeys)
|
|
require.Len(t, page1.Version, maxKeys)
|
|
checkVersionsNames(t, page1, objects)
|
|
require.Equal(t, page1.Version[maxKeys-1].VersionID, page1.NextVersionIDMarker)
|
|
require.True(t, page1.IsTruncated)
|
|
|
|
page2 := listObjectsVersions(hc, bktName, "", "", page1.NextKeyMarker, page1.NextVersionIDMarker, maxKeys)
|
|
require.Len(t, page2.Version, maxKeys)
|
|
checkVersionsNames(t, page1, objects)
|
|
require.Empty(t, page2.NextVersionIDMarker)
|
|
require.False(t, page2.IsTruncated)
|
|
}
|
|
|
|
func checkVersionsNames(t *testing.T, versions *ListObjectsVersionsResponse, names []string) {
|
|
for i, v := range versions.Version {
|
|
require.Equal(t, names[i], v.Key)
|
|
}
|
|
}
|
|
|
|
func listObjectsV2(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
|
|
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
|
if len(startAfter) != 0 {
|
|
query.Add("start-after", startAfter)
|
|
}
|
|
if len(continuationToken) != 0 {
|
|
query.Add("continuation-token", continuationToken)
|
|
}
|
|
|
|
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
|
hc.Handler().ListObjectsV2Handler(w, r)
|
|
assertStatus(hc.t, w, http.StatusOK)
|
|
res := &ListObjectsV2Response{}
|
|
parseTestResponse(hc.t, w, res)
|
|
return res
|
|
}
|
|
|
|
func validateListV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, continuationToken string, maxKeys int,
|
|
isTruncated, last bool, checkObjects, checkPrefixes []string) string {
|
|
response := listObjectsV2(tc, bktName, prefix, delimiter, "", continuationToken, maxKeys)
|
|
|
|
require.Equal(t, isTruncated, response.IsTruncated)
|
|
require.Equal(t, last, len(response.NextContinuationToken) == 0)
|
|
|
|
require.Len(t, response.Contents, len(checkObjects))
|
|
for i := 0; i < len(checkObjects); i++ {
|
|
require.Equal(t, checkObjects[i], response.Contents[i].Key)
|
|
}
|
|
|
|
require.Len(t, response.CommonPrefixes, len(checkPrefixes))
|
|
for i := 0; i < len(checkPrefixes); i++ {
|
|
require.Equal(t, checkPrefixes[i], response.CommonPrefixes[i].Prefix)
|
|
}
|
|
|
|
return response.NextContinuationToken
|
|
}
|
|
|
|
func prepareCommonListObjectsQuery(prefix, delimiter string, maxKeys int) url.Values {
|
|
query := make(url.Values)
|
|
|
|
if len(delimiter) != 0 {
|
|
query.Add("delimiter", delimiter)
|
|
}
|
|
if len(prefix) != 0 {
|
|
query.Add("prefix", prefix)
|
|
}
|
|
if maxKeys != -1 {
|
|
query.Add("max-keys", strconv.Itoa(maxKeys))
|
|
}
|
|
|
|
return query
|
|
}
|
|
|
|
func listObjectsV1(hc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int) *ListObjectsV1Response {
|
|
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
|
if len(marker) != 0 {
|
|
query.Add("marker", marker)
|
|
}
|
|
|
|
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
|
hc.Handler().ListObjectsV1Handler(w, r)
|
|
assertStatus(hc.t, w, http.StatusOK)
|
|
res := &ListObjectsV1Response{}
|
|
parseTestResponse(hc.t, w, res)
|
|
return res
|
|
}
|
|
|
|
func listObjectsVersions(hc *handlerContext, bktName, prefix, delimiter, keyMarker, versionIDMarker string, maxKeys int) *ListObjectsVersionsResponse {
|
|
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
|
if len(keyMarker) != 0 {
|
|
query.Add("key-marker", keyMarker)
|
|
}
|
|
if len(versionIDMarker) != 0 {
|
|
query.Add("version-id-marker", versionIDMarker)
|
|
}
|
|
|
|
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
|
hc.Handler().ListBucketObjectVersionsHandler(w, r)
|
|
assertStatus(hc.t, w, http.StatusOK)
|
|
res := &ListObjectsVersionsResponse{}
|
|
parseTestResponse(hc.t, w, res)
|
|
return res
|
|
}
|