TrueCloudLab/frostfs-s3-gw
18
3
Fork 20
Code Issues 29 Pull requests 8 Projects Releases 23 Packages Activity Actions

[#165] Extend error on getting listing containers not in current namespace

Browse source 
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
Denis Kirillov 2024-01-19 15:15:15 +03:00
parent 739a6ec9df
commit cf4fc3b602
3 changed files with 114 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

109
api/handler/object_list_test.go
View file

@ -1,6 +1,7 @@
package handler
import (
"fmt"
"net/http"
"net/url"
"sort"
@ -335,6 +336,35 @@ func TestS3BucketListV2DelimiterPercentage(t *testing.T) {
require.Equal(t, "c%", listV2Response.CommonPrefixes[1].Prefix)
}
func TestS3BucketListDelimiterPrefix(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-for-listing"
bktInfo := createTestBucket(hc, bktName)
objects := []string{"asdf", "boo/bar", "boo/baz/xyzzy", "cquux/thud", "cquux/bla"}
for _, objName := range objects {
createTestObject(hc, bktInfo, objName, encryption.Params{})
}
var empty []string
delim := "/"
prefix := ""
marker := validateListV1(t, hc, bktName, prefix, delim, "", 1, true, []string{"asdf"}, empty, "asdf")
marker = validateListV1(t, hc, bktName, prefix, delim, marker, 1, true, empty, []string{"boo/"}, "boo/")
validateListV1(t, hc, bktName, prefix, delim, marker, 1, false, empty, []string{"cquux/"}, "")
marker = validateListV1(t, hc, bktName, prefix, delim, "", 2, true, []string{"asdf"}, []string{"boo/"}, "boo/")
validateListV1(t, hc, bktName, prefix, delim, marker, 2, false, empty, []string{"cquux/"}, "")
prefix = "boo/"
marker = validateListV1(t, hc, bktName, prefix, delim, "", 1, true, []string{"boo/bar"}, empty, "boo/bar")
validateListV1(t, hc, bktName, prefix, delim, marker, 1, false, empty, []string{"boo/baz/"}, "")
validateListV1(t, hc, bktName, prefix, delim, "", 2, false, []string{"boo/bar"}, []string{"boo/baz/"}, "")
}
func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
tc := prepareHandlerContext(t)
@ -364,6 +394,65 @@ func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
validateListV2(t, tc, bktName, prefix, delim, "", 2, false, true, []string{"boo/bar"}, []string{"boo/baz/"})
}
func TestS3BucketListDelimiterPrefixUnderscore(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-for-listing"
bktInfo := createTestBucket(hc, bktName)
objects := []string{"_obj1_", "_under1/bar", "_under1/baz/xyzzy", "_under2/thud", "_under2/bla"}
for _, objName := range objects {
createTestObject(hc, bktInfo, objName, encryption.Params{})
}
var empty []string
delim := "/"
prefix := ""
marker := validateListV1(t, hc, bktName, prefix, delim, "", 1, true, []string{"_obj1_"}, empty, "_obj1_")
marker = validateListV1(t, hc, bktName, prefix, delim, marker, 1, true, empty, []string{"_under1/"}, "_under1/")
validateListV1(t, hc, bktName, prefix, delim, marker, 1, false, empty, []string{"_under2/"}, "")
marker = validateListV1(t, hc, bktName, prefix, delim, "", 2, true, []string{"_obj1_"}, []string{"_under1/"}, "_under1/")
validateListV1(t, hc, bktName, prefix, delim, marker, 2, false, empty, []string{"_under2/"}, "")
prefix = "_under1/"
marker = validateListV1(t, hc, bktName, prefix, delim, "", 1, true, []string{"_under1/bar"}, empty, "_under1/bar")
validateListV1(t, hc, bktName, prefix, delim, marker, 1, false, empty, []string{"_under1/baz/"}, "")
validateListV1(t, hc, bktName, prefix, delim, "", 2, false, []string{"_under1/bar"}, []string{"_under1/baz/"}, "")
}
func TestS3BucketListDelimiterNotSkipSpecial(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-for-listing"
bktInfo := createTestBucket(hc, bktName)
objects := []string{"0/"}
for i := 1000; i < 1999; i++ {
objects = append(objects, fmt.Sprintf("0/%d", i))
}
objects2 := []string{"1999", "1999#", "1999+", "2000"}
objects = append(objects, objects2...)
for _, objName := range objects {
createTestObject(hc, bktInfo, objName, encryption.Params{})
}
delimiter := "/"
list := listObjectsV1(hc, bktName, "", delimiter, "", -1)
require.Equal(t, delimiter, list.Delimiter)
require.Equal(t, []CommonPrefix{{Prefix: "0/"}}, list.CommonPrefixes)
require.Len(t, list.Contents, len(objects2))
for i := 0; i < len(list.Contents); i++ {
require.Equal(t, objects2[i], list.Contents[i].Key)
}
}
func TestMintVersioningListObjectVersionsVersionIDContinuation(t *testing.T) {
hc := prepareHandlerContext(t)
@ -482,6 +571,26 @@ func listObjectsV2Ext(hc *handlerContext, bktName, prefix, delimiter, startAfter
return res
}
func validateListV1(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int,
isTruncated bool, checkObjects, checkPrefixes []string, nextMarker string) string {
response := listObjectsV1(tc, bktName, prefix, delimiter, marker, maxKeys)
require.Equal(t, isTruncated, response.IsTruncated)
require.Equal(t, nextMarker, response.NextMarker)
require.Len(t, response.Contents, len(checkObjects))
for i := 0; i < len(checkObjects); i++ {
require.Equal(t, checkObjects[i], response.Contents[i].Key)
}
require.Len(t, response.CommonPrefixes, len(checkPrefixes))
for i := 0; i < len(checkPrefixes); i++ {
require.Equal(t, checkPrefixes[i], response.CommonPrefixes[i].Prefix)
}
return response.NextMarker
}
func validateListV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, continuationToken string, maxKeys int,
isTruncated, last bool, checkObjects, checkPrefixes []string) string {
response := listObjectsV2(tc, bktName, prefix, delimiter, "", continuationToken, maxKeys)

2
api/layer/container.go
View file

@ -76,7 +76,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
zone, _ := n.features.FormContainerZone(reqInfo.Namespace)
if zone != info.Zone {
return nil, fmt.Errorf("ns '%s' and zone '%s' are mismatched", zone, info.Zone)
return nil, fmt.Errorf("ns '%s' and zone '%s' are mismatched for container '%s'", zone, info.Zone, idCnr)
}
n.cache.PutBucket(info)

10
api/layer/listing.go
View file

@ -296,11 +296,6 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p allObjectParams)
return nil, nil, fmt.Errorf("failed to get next object from stream: %w", err)
}
// probably isn't necessary
sort.Slice(objects, func(i, j int) bool {
return objects[i].FilePath < objects[j].FilePath
})
if len(objects) > p.MaxKeys {
next = objects[p.MaxKeys]
objects = objects[:p.MaxKeys]
@ -523,7 +518,10 @@ func (n *layer) initWorkerPoolStream(ctx context.Context, size int, p allObjectP
default:
}
if node.IsFilledExtra() || tryDirectoryName(node, p.Prefix, p.Delimiter) != "" { // todo think to not compute twice
if dirName := tryDirectoryName(node, p.Prefix, p.Delimiter); dirName != "" || node.IsFilledExtra() { // todo think to not compute twice
if dirName != "" {
node.FilePath = dirName
}
select {
case <-ctx.Done():
case objCh <- node: