TrueCloudLab/frostfs-node
20
1
Fork 28
Code Issues 107 Pull requests 4 Projects Releases 2 Packages 4 Activity Actions

[#1412] metabase: Add search by indexed attributes

Browse source 
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2024-10-02 13:04:29 +03:00
parent be744ae3e6
commit 1efa64ee72
Signed by: dstepanov-yadro
GPG key ID: 237AF1A763293BC0
3 changed files with 169 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

81
pkg/local_object_storage/metabase/select_test.go
View file

@ -26,6 +26,16 @@ import (
func TestDB_SelectUserAttributes(t *testing.T) {
t.Parallel()
t.Run("with_index", func(t *testing.T) {
testSelectUserAttributes(t, true)
})
t.Run("without_index", func(t *testing.T) {
testSelectUserAttributes(t, false)
})
}
func testSelectUserAttributes(t *testing.T, index bool) {
t.Parallel()
db := newDB(t)
defer func() { require.NoError(t, db.Close()) }()
@ -36,44 +46,52 @@ func TestDB_SelectUserAttributes(t *testing.T) {
testutil.AddAttribute(raw1, "foo", "bar")
testutil.AddAttribute(raw1, "x", "y")
err := putBig(db, raw1)
var putPrm meta.PutPrm
putPrm.SetIndexAttributes(index)
putPrm.SetObject(raw1)
_, err := db.Put(context.Background(), putPrm)
require.NoError(t, err)
raw2 := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(raw2, "foo", "bar")
testutil.AddAttribute(raw2, "x", "z")
err = putBig(db, raw2)
putPrm.SetObject(raw2)
_, err = db.Put(context.Background(), putPrm)
require.NoError(t, err)
raw3 := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(raw3, "a", "b")
err = putBig(db, raw3)
putPrm.SetObject(raw3)
_, err = db.Put(context.Background(), putPrm)
require.NoError(t, err)
raw4 := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(raw4, "path", "test/1/2")
testutil.AddAttribute(raw4, objectSDK.AttributeFilePath, "/test/1/2")
err = putBig(db, raw4)
putPrm.SetObject(raw4)
_, err = db.Put(context.Background(), putPrm)
require.NoError(t, err)
raw5 := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(raw5, "path", "test/1/3")
testutil.AddAttribute(raw5, objectSDK.AttributeFilePath, "/test/1/3")
err = putBig(db, raw5)
putPrm.SetObject(raw5)
_, err = db.Put(context.Background(), putPrm)
require.NoError(t, err)
raw6 := testutil.GenerateObjectWithCID(cnr)
testutil.AddAttribute(raw6, "path", "test/2/3")
testutil.AddAttribute(raw6, objectSDK.AttributeFilePath, "/test/2/3")
err = putBig(db, raw6)
putPrm.SetObject(raw6)
_, err = db.Put(context.Background(), putPrm)
require.NoError(t, err)
raw7 := testutil.GenerateObjectWithCID(cnr)
var attr objectSDK.Attribute
attr.SetKey("path")
attr.SetValue("test/3/4")
attr.SetKey(objectSDK.AttributeFilePath)
attr.SetValue("/test/3/4")
attrs := raw7.Attributes()
attrs = append(attrs, attr)
ech := objectSDK.NewECHeader(objectSDK.ECParentInfo{
@ -81,37 +99,39 @@ func TestDB_SelectUserAttributes(t *testing.T) {
Attributes: attrs,
}, 0, 3, []byte{}, 0)
raw7.SetECHeader(ech)
require.NoError(t, putBig(db, raw7))
putPrm.SetObject(raw7)
_, err = db.Put(context.Background(), putPrm)
require.NoError(t, err)
var raw7Parent oid.Address
raw7Parent.SetContainer(cnr)
raw7Parent.SetObject(ech.Parent())
fs := objectSDK.SearchFilters{}
fs.AddFilter("foo", "bar", objectSDK.MatchStringEqual)
testSelect(t, db, cnr, fs,
testSelect2(t, db, cnr, fs, index,
object.AddressOf(raw1),
object.AddressOf(raw2),
)
fs = objectSDK.SearchFilters{}
fs.AddFilter("x", "y", objectSDK.MatchStringEqual)
testSelect(t, db, cnr, fs, object.AddressOf(raw1))
testSelect2(t, db, cnr, fs, index, object.AddressOf(raw1))
fs = objectSDK.SearchFilters{}
fs.AddFilter("x", "y", objectSDK.MatchStringNotEqual)
testSelect(t, db, cnr, fs, object.AddressOf(raw2))
testSelect2(t, db, cnr, fs, index, object.AddressOf(raw2))
fs = objectSDK.SearchFilters{}
fs.AddFilter("a", "b", objectSDK.MatchStringEqual)
testSelect(t, db, cnr, fs, object.AddressOf(raw3))
testSelect2(t, db, cnr, fs, index, object.AddressOf(raw3))
fs = objectSDK.SearchFilters{}
fs.AddFilter("c", "d", objectSDK.MatchStringEqual)
testSelect(t, db, cnr, fs)
testSelect2(t, db, cnr, fs, index)
fs = objectSDK.SearchFilters{}
fs.AddFilter("foo", "", objectSDK.MatchNotPresent)
testSelect(t, db, cnr, fs,
testSelect2(t, db, cnr, fs, index,
object.AddressOf(raw3),
object.AddressOf(raw4),
object.AddressOf(raw5),
@ -121,7 +141,7 @@ func TestDB_SelectUserAttributes(t *testing.T) {
fs = objectSDK.SearchFilters{}
fs.AddFilter("a", "", objectSDK.MatchNotPresent)
testSelect(t, db, cnr, fs,
testSelect2(t, db, cnr, fs, index,
object.AddressOf(raw1),
object.AddressOf(raw2),
object.AddressOf(raw4),
@ -131,7 +151,7 @@ func TestDB_SelectUserAttributes(t *testing.T) {
)
fs = objectSDK.SearchFilters{}
testSelect(t, db, cnr, fs,
testSelect2(t, db, cnr, fs, index,
object.AddressOf(raw1),
object.AddressOf(raw2),
object.AddressOf(raw3),
@ -143,7 +163,7 @@ func TestDB_SelectUserAttributes(t *testing.T) {
fs = objectSDK.SearchFilters{}
fs.AddFilter("key", "", objectSDK.MatchNotPresent)
testSelect(t, db, cnr, fs,
testSelect2(t, db, cnr, fs, index,
object.AddressOf(raw1),
object.AddressOf(raw2),
object.AddressOf(raw3),
@ -154,8 +174,8 @@ func TestDB_SelectUserAttributes(t *testing.T) {
)
fs = objectSDK.SearchFilters{}
fs.AddFilter("path", "test", objectSDK.MatchCommonPrefix)
testSelect(t, db, cnr, fs,
fs.AddFilter(objectSDK.AttributeFilePath, "/test", objectSDK.MatchCommonPrefix)
testSelect2(t, db, cnr, fs, index,
object.AddressOf(raw4),
object.AddressOf(raw5),
object.AddressOf(raw6),
@ -163,15 +183,15 @@ func TestDB_SelectUserAttributes(t *testing.T) {
)
fs = objectSDK.SearchFilters{}
fs.AddFilter("path", "test/1", objectSDK.MatchCommonPrefix)
testSelect(t, db, cnr, fs,
fs.AddFilter(objectSDK.AttributeFilePath, "/test/1", objectSDK.MatchCommonPrefix)
testSelect2(t, db, cnr, fs, index,
object.AddressOf(raw4),
object.AddressOf(raw5),
)
fs = objectSDK.SearchFilters{}
fs.AddFilter("path", "test/3/4", objectSDK.MatchStringEqual)
testSelect(t, db, cnr, fs,
fs.AddFilter(objectSDK.AttributeFilePath, "/test/3/4", objectSDK.MatchStringEqual)
testSelect2(t, db, cnr, fs, index,
raw7Parent,
)
}
@ -1185,11 +1205,11 @@ func TestExpiredObjects(t *testing.T) {
cidExp, _ := exp.ContainerID()
cidNonExp, _ := nonExp.ContainerID()
objs, err := metaSelect(db, cidExp, objectSDK.SearchFilters{})
objs, err := metaSelect(db, cidExp, objectSDK.SearchFilters{}, false)
require.NoError(t, err)
require.Empty(t, objs) // expired object should not be returned
objs, err = metaSelect(db, cidNonExp, objectSDK.SearchFilters{})
objs, err = metaSelect(db, cidNonExp, objectSDK.SearchFilters{}, false)
require.NoError(t, err)
require.NotEmpty(t, objs)
})
@ -1211,10 +1231,11 @@ func benchmarkSelect(b *testing.B, db *meta.DB, cid cidSDK.ID, fs objectSDK.Sear
}
}
func metaSelect(db *meta.DB, cnr cidSDK.ID, fs objectSDK.SearchFilters) ([]oid.Address, error) {
func metaSelect(db *meta.DB, cnr cidSDK.ID, fs objectSDK.SearchFilters, useAttributeIndex bool) ([]oid.Address, error) {
var prm meta.SelectPrm
prm.SetFilters(fs)
prm.SetContainerID(cnr)
prm.SetUseAttributeIndex(useAttributeIndex)
res, err := db.Select(context.Background(), prm)
return res.AddressList(), err