forked from TrueCloudLab/frostfs-s3-gw
[#165] Support streaming listing
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
parent
84af85ed67
commit
29ac91dfd5
12 changed files with 938 additions and 39 deletions
|
|
@ -1,13 +1,17 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -162,6 +166,120 @@ func TestS3BucketListDelimiterBasic(t *testing.T) {
|
|||
require.Equal(t, "quux/", listV1Response.CommonPrefixes[1].Prefix)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2PrefixAlt(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"bar", "baz", "foo"}
|
||||
for _, objName := range objects {
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
response := listObjectsV2(hc, bktName, "ba", "", "", "", -1)
|
||||
|
||||
require.Equal(t, "ba", response.Prefix)
|
||||
require.Len(t, response.Contents, 2)
|
||||
require.Equal(t, "bar", response.Contents[0].Key)
|
||||
require.Equal(t, "baz", response.Contents[1].Key)
|
||||
require.Empty(t, response.CommonPrefixes)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2PrefixNotExist(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"foo/bar", "foo/baz", "quux"}
|
||||
for _, objName := range objects {
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
response := listObjectsV2(hc, bktName, "d", "", "", "", -1)
|
||||
|
||||
require.Equal(t, "d", response.Prefix)
|
||||
require.Empty(t, response.Contents)
|
||||
require.Empty(t, response.CommonPrefixes)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2PrefixUnreadable(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"foo/bar", "foo/baz", "quux"}
|
||||
for _, objName := range objects {
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
response := listObjectsV2(hc, bktName, "\x0a", "", "", "", -1)
|
||||
|
||||
require.Equal(t, "\x0a", response.Prefix)
|
||||
require.Empty(t, response.Contents)
|
||||
require.Empty(t, response.CommonPrefixes)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2PrefixDelimiterAlt(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"bar", "bazar", "cab", "foo"}
|
||||
for _, objName := range objects {
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
response := listObjectsV2(hc, bktName, "ba", "a", "", "", -1)
|
||||
|
||||
require.Equal(t, "ba", response.Prefix)
|
||||
require.Equal(t, "a", response.Delimiter)
|
||||
require.Len(t, response.Contents, 1)
|
||||
require.Equal(t, "bar", response.Contents[0].Key)
|
||||
require.Len(t, response.CommonPrefixes, 1)
|
||||
require.Equal(t, "baza", response.CommonPrefixes[0].Prefix)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2PrefixDelimiterDelimiterNotExist(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"b/a/c", "b/a/g", "b/a/r", "g"}
|
||||
for _, objName := range objects {
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
response := listObjectsV2(hc, bktName, "b", "z", "", "", -1)
|
||||
|
||||
require.Len(t, response.Contents, 3)
|
||||
require.Equal(t, "b/a/c", response.Contents[0].Key)
|
||||
require.Equal(t, "b/a/g", response.Contents[1].Key)
|
||||
require.Equal(t, "b/a/r", response.Contents[2].Key)
|
||||
require.Empty(t, response.CommonPrefixes)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2PrefixDelimiterPrefixDelimiterNotExist(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objects := []string{"b/a/c", "b/a/g", "b/a/r", "g"}
|
||||
for _, objName := range objects {
|
||||
putObject(hc, bktName, objName)
|
||||
}
|
||||
|
||||
response := listObjectsV2(hc, bktName, "y", "z", "", "", -1)
|
||||
|
||||
require.Empty(t, response.Contents)
|
||||
require.Empty(t, response.CommonPrefixes)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2DelimiterPercentage(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
|
|
@ -250,7 +368,148 @@ func checkVersionsNames(t *testing.T, versions *ListObjectsVersionsResponse, nam
|
|||
}
|
||||
}
|
||||
|
||||
func TestHugeListV2(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listingv2"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
objects := prepareObjects(hc, bktInfo, "", 50005)
|
||||
|
||||
fmt.Println("listing start")
|
||||
start := time.Now()
|
||||
|
||||
resp := &ListObjectsV2Response{IsTruncated: true}
|
||||
for resp.IsTruncated {
|
||||
resp = listObjectsV2(hc, bktName, "", "", "", resp.NextContinuationToken, -1)
|
||||
for i, content := range resp.Contents {
|
||||
if content.Key != objects[i] {
|
||||
t.Errorf("expected '%s', got '%s'", objects[i], content.Key)
|
||||
}
|
||||
}
|
||||
objects = objects[len(resp.Contents):]
|
||||
}
|
||||
require.Empty(t, objects)
|
||||
|
||||
fmt.Println(time.Since(start))
|
||||
}
|
||||
|
||||
func TestListV2StreamNested1(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listingv2-nested"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
objects1 := prepareObjects(hc, bktInfo, "prefix", 10)
|
||||
objects2 := prepareObjects(hc, bktInfo, "prefix2", 10)
|
||||
|
||||
objects := append(objects1, objects2...)
|
||||
|
||||
fmt.Println("listing start")
|
||||
start := time.Now()
|
||||
|
||||
resp := &ListObjectsV2Response{IsTruncated: true}
|
||||
for resp.IsTruncated {
|
||||
resp = listObjectsV2(hc, bktName, "", "", "", resp.NextContinuationToken, -1)
|
||||
for i, content := range resp.Contents {
|
||||
if content.Key != objects[i] {
|
||||
t.Errorf("expected '%s', got '%s'", objects[i], content.Key)
|
||||
}
|
||||
}
|
||||
objects = objects[len(resp.Contents):]
|
||||
}
|
||||
require.Empty(t, objects)
|
||||
|
||||
fmt.Println(time.Since(start))
|
||||
}
|
||||
|
||||
func TestHugeListV1(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listingv1"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
objects := prepareObjects(hc, bktInfo, "", 50005)
|
||||
|
||||
fmt.Println("listing start")
|
||||
start := time.Now()
|
||||
|
||||
resp := &ListObjectsV1Response{IsTruncated: true}
|
||||
for resp.IsTruncated {
|
||||
resp = listObjectsV1(hc, bktName, "", "", resp.NextMarker, -1)
|
||||
for i, content := range resp.Contents {
|
||||
if content.Key != objects[i] {
|
||||
t.Errorf("expected '%s', got '%s'", objects[i], content.Key)
|
||||
}
|
||||
}
|
||||
objects = objects[len(resp.Contents):]
|
||||
}
|
||||
|
||||
require.Empty(t, objects)
|
||||
|
||||
fmt.Println(time.Since(start))
|
||||
}
|
||||
|
||||
func prepareObjects(hc *handlerContext, bktInfo *data.BucketInfo, prefix string, size int) []string {
|
||||
treeID := "version"
|
||||
parentID := uint64(0)
|
||||
if prefix != "" {
|
||||
for _, filename := range strings.Split(prefix, "/") {
|
||||
nodeID, err := hc.treeMock.AddNode(hc.Context(), bktInfo, treeID, parentID, map[string]string{
|
||||
"FileName": filename,
|
||||
})
|
||||
require.NoError(hc.t, err)
|
||||
parentID = nodeID
|
||||
}
|
||||
prefix += "/"
|
||||
}
|
||||
|
||||
objects := make([]string, size)
|
||||
|
||||
for i := range objects {
|
||||
filename := "object" + strconv.Itoa(i)
|
||||
filepath := prefix + filename
|
||||
|
||||
prm := layer.PrmObjectCreate{
|
||||
Container: bktInfo.CID,
|
||||
Filepath: filepath,
|
||||
Payload: nil,
|
||||
}
|
||||
|
||||
id, err := hc.tp.CreateObject(hc.Context(), prm)
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
newVersion := &data.NodeVersion{
|
||||
BaseNodeVersion: data.BaseNodeVersion{
|
||||
OID: id,
|
||||
ETag: "12345678",
|
||||
FilePath: filepath,
|
||||
},
|
||||
IsUnversioned: true,
|
||||
IsCombined: false,
|
||||
}
|
||||
|
||||
_, err = hc.treeMock.AddNodeBase(hc.Context(), bktInfo, treeID, parentID, map[string]string{
|
||||
"OID": newVersion.OID.EncodeToString(),
|
||||
"FileName": filename,
|
||||
"IsUnversioned": "true",
|
||||
}, false)
|
||||
require.NoError(hc.t, err)
|
||||
objects[i] = filepath
|
||||
}
|
||||
|
||||
hc.treeMock.Sort()
|
||||
|
||||
sort.Strings(objects)
|
||||
|
||||
return objects
|
||||
}
|
||||
|
||||
func listObjectsV2(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
|
||||
return listObjectsV2Ext(hc, bktName, prefix, delimiter, startAfter, continuationToken, "", maxKeys)
|
||||
}
|
||||
|
||||
func listObjectsV2Ext(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken, encodingType string, maxKeys int) *ListObjectsV2Response {
|
||||
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
||||
if len(startAfter) != 0 {
|
||||
query.Add("start-after", startAfter)
|
||||
|
|
@ -258,6 +517,9 @@ func listObjectsV2(hc *handlerContext, bktName, prefix, delimiter, startAfter, c
|
|||
if len(continuationToken) != 0 {
|
||||
query.Add("continuation-token", continuationToken)
|
||||
}
|
||||
if len(encodingType) != 0 {
|
||||
query.Add("encoding-type", encodingType)
|
||||
}
|
||||
|
||||
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
||||
hc.Handler().ListObjectsV2Handler(w, r)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue