realloc/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#616] Reduce number of requests during listing

Browse source 
Check if object is directory before request to NeoFS

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-08-13 13:03:49 +03:00 • committed by Kirillov Denis
parent 107d8a9033
commit 3824151699
5 changed files with 135 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

78
api/layer/util_test.go
View file

@ -3,14 +3,12 @@ package layer
import (
"encoding/hex"
"net/http"
"strconv"
"testing"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-sdk-go/checksum"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"github.com/stretchr/testify/require"
@ -23,30 +21,6 @@ var (
defaultTestContentType = http.DetectContentType(defaultTestPayload)
)
func newTestObject(id oid.ID, bkt *data.BucketInfo, name string) *object.Object {
filename := object.NewAttribute()
filename.SetKey(object.AttributeFileName)
filename.SetValue(name)
created := object.NewAttribute()
created.SetKey(object.AttributeTimestamp)
created.SetValue(strconv.FormatInt(defaultTestCreated.Unix(), 10))
contentType := object.NewAttribute()
contentType.SetKey(object.AttributeContentType)
contentType.SetValue(defaultTestContentType)
obj := object.New()
obj.SetID(id)
obj.SetOwnerID(&bkt.Owner)
obj.SetContainerID(bkt.CID)
obj.SetPayload(defaultTestPayload)
obj.SetAttributes(*filename, *created, *contentType)
obj.SetPayloadSize(uint64(defaultTestPayloadLength))
return obj
}
func newTestInfo(obj oid.ID, bkt *data.BucketInfo, name string, isDir bool) *data.ObjectInfo {
var hashSum checksum.Checksum
info := &data.ObjectInfo{
@ -72,7 +46,16 @@ func newTestInfo(obj oid.ID, bkt *data.BucketInfo, name string, isDir bool) *dat
return info
}
func Test_objectInfoName(t *testing.T) {
func newTestNodeVersion(id oid.ID, name string) *data.NodeVersion {
return &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
OID: id,
FilePath: name,
},
}
}
func TestTryDirectory(t *testing.T) {
var uid user.ID
var id oid.ID
var containerID cid.ID
@ -88,77 +71,82 @@ func Test_objectInfoName(t *testing.T) {
name string
prefix string
result *data.ObjectInfo
object *object.Object
node *data.NodeVersion
delimiter string
}{
{
name: "small.jpg",
result: newTestInfo(id, bkt, "small.jpg", false),
object: newTestObject(id, bkt, "small.jpg"),
result: nil,
node: newTestNodeVersion(id, "small.jpg"),
},
{
name: "small.jpg not matched prefix",
prefix: "big",
result: nil,
object: newTestObject(id, bkt, "small.jpg"),
node: newTestNodeVersion(id, "small.jpg"),
},
{
name: "small.jpg delimiter",
delimiter: "/",
result: newTestInfo(id, bkt, "small.jpg", false),
object: newTestObject(id, bkt, "small.jpg"),
result: nil,
node: newTestNodeVersion(id, "small.jpg"),
},
{
name: "test/small.jpg",
result: newTestInfo(id, bkt, "test/small.jpg", false),
object: newTestObject(id, bkt, "test/small.jpg"),
result: nil,
node: newTestNodeVersion(id, "test/small.jpg"),
},
{
name: "test/small.jpg with prefix and delimiter",
prefix: "test/",
delimiter: "/",
result: newTestInfo(id, bkt, "test/small.jpg", false),
object: newTestObject(id, bkt, "test/small.jpg"),
result: nil,
node: newTestNodeVersion(id, "test/small.jpg"),
},
{
name: "a/b/small.jpg",
prefix: "a",
result: newTestInfo(id, bkt, "a/b/small.jpg", false),
object: newTestObject(id, bkt, "a/b/small.jpg"),
result: nil,
node: newTestNodeVersion(id, "a/b/small.jpg"),
},
{
name: "a/b/small.jpg",
prefix: "a/",
delimiter: "/",
result: newTestInfo(id, bkt, "a/b/", true),
object: newTestObject(id, bkt, "a/b/small.jpg"),
node: newTestNodeVersion(id, "a/b/small.jpg"),
},
{
name: "a/b/c/small.jpg",
prefix: "a/",
delimiter: "/",
result: newTestInfo(id, bkt, "a/b/", true),
object: newTestObject(id, bkt, "a/b/c/small.jpg"),
node: newTestNodeVersion(id, "a/b/c/small.jpg"),
},
{
name: "a/b/c/small.jpg",
prefix: "a/b/c/s",
delimiter: "/",
result: newTestInfo(id, bkt, "a/b/c/small.jpg", false),
object: newTestObject(id, bkt, "a/b/c/small.jpg"),
result: nil,
node: newTestNodeVersion(id, "a/b/c/small.jpg"),
},
{
name: "a/b/c/big.jpg",
prefix: "a/b/",
delimiter: "/",
result: newTestInfo(id, bkt, "a/b/c/", true),
object: newTestObject(id, bkt, "a/b/c/big.jpg"),
node: newTestNodeVersion(id, "a/b/c/big.jpg"),
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
info := processObjectInfoName(objectInfoFromMeta(bkt, tc.object), tc.prefix, tc.delimiter)
info := tryDirectory(bkt, tc.node, tc.prefix, tc.delimiter)
if tc.result != nil {
tc.result.Created = time.Time{}
tc.result.Owner = user.ID{}
}
require.Equal(t, tc.result, info)
})
}