[#1412] searchSvc: Check container is indexed

For non S3 containers it is expected to use attributes index for some
attributes.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2024-10-02 14:52:54 +03:00
parent 1efa64ee72
commit 4572fa4874
Signed by: dstepanov-yadro
GPG key ID: 237AF1A763293BC0
9 changed files with 41 additions and 16 deletions

View file

@ -174,7 +174,7 @@ func initObjectService(c *cfg) {
sPutV2 := createPutSvcV2(sPut, keyStorage) sPutV2 := createPutSvcV2(sPut, keyStorage)
sSearch := createSearchSvc(c, keyStorage, traverseGen, c.clientCache) sSearch := createSearchSvc(c, keyStorage, traverseGen, c.clientCache, c.cfgObject.cnrSource)
sSearchV2 := createSearchSvcV2(sSearch, keyStorage) sSearchV2 := createSearchSvcV2(sSearch, keyStorage)
@ -366,7 +366,7 @@ func createPatchSvc(sGet *getsvc.Service, sPut *putsvc.Service) *patchsvc.Servic
return patchsvc.NewService(sPut.Config, sGet) return patchsvc.NewService(sPut.Config, sGet)
} }
func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache) *searchsvc.Service { func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache, containerSource containercore.Source) *searchsvc.Service {
ls := c.cfgObject.cfgLocalStorage.localStorage ls := c.cfgObject.cfgLocalStorage.localStorage
return searchsvc.New( return searchsvc.New(
@ -377,6 +377,7 @@ func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Trav
), ),
c.netMapSource, c.netMapSource,
keyStorage, keyStorage,
containerSource,
searchsvc.WithLogger(c.log), searchsvc.WithLogger(c.log),
) )
} }

View file

@ -143,7 +143,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo
var selectPrm shard.SelectPrm var selectPrm shard.SelectPrm
selectPrm.SetFilters(fs) selectPrm.SetFilters(fs)
selectPrm.SetContainerID(addr.Container()) selectPrm.SetContainerID(addr.Container(), false) // doesn't matter for search by splitID
var inhumePrm shard.InhumePrm var inhumePrm shard.InhumePrm
if force { if force {

View file

@ -49,7 +49,7 @@ func TestStorageEngine_Inhume(t *testing.T) {
_, err = e.Inhume(context.Background(), inhumePrm) _, err = e.Inhume(context.Background(), inhumePrm)
require.NoError(t, err) require.NoError(t, err)
addrs, err := Select(context.Background(), e, cnr, fs) addrs, err := Select(context.Background(), e, cnr, false, fs)
require.NoError(t, err) require.NoError(t, err)
require.Empty(t, addrs) require.Empty(t, addrs)
}) })
@ -78,7 +78,7 @@ func TestStorageEngine_Inhume(t *testing.T) {
_, err = e.Inhume(context.Background(), inhumePrm) _, err = e.Inhume(context.Background(), inhumePrm)
require.NoError(t, err) require.NoError(t, err)
addrs, err := Select(context.Background(), e, cnr, fs) addrs, err := Select(context.Background(), e, cnr, false, fs)
require.NoError(t, err) require.NoError(t, err)
require.Empty(t, addrs) require.Empty(t, addrs)
}) })

View file

@ -14,8 +14,9 @@ import (
// SelectPrm groups the parameters of Select operation. // SelectPrm groups the parameters of Select operation.
type SelectPrm struct { type SelectPrm struct {
cnr cid.ID cnr cid.ID
filters objectSDK.SearchFilters filters objectSDK.SearchFilters
indexedContainer bool
} }
// SelectRes groups the resulting values of Select operation. // SelectRes groups the resulting values of Select operation.
@ -24,8 +25,9 @@ type SelectRes struct {
} }
// WithContainerID is a Select option to set the container id to search in. // WithContainerID is a Select option to set the container id to search in.
func (p *SelectPrm) WithContainerID(cnr cid.ID) { func (p *SelectPrm) WithContainerID(cnr cid.ID, indexedContainer bool) {
p.cnr = cnr p.cnr = cnr
p.indexedContainer = indexedContainer
} }
// WithFilters is a Select option to set the object filters. // WithFilters is a Select option to set the object filters.
@ -67,7 +69,7 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes,
var outError error var outError error
var shPrm shard.SelectPrm var shPrm shard.SelectPrm
shPrm.SetContainerID(prm.cnr) shPrm.SetContainerID(prm.cnr, prm.indexedContainer)
shPrm.SetFilters(prm.filters) shPrm.SetFilters(prm.filters)
e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) {
@ -140,9 +142,9 @@ func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, erro
} }
// Select selects objects from local storage using provided filters. // Select selects objects from local storage using provided filters.
func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs objectSDK.SearchFilters) ([]oid.Address, error) { func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, isIndexedContainer bool, fs objectSDK.SearchFilters) ([]oid.Address, error) {
var selectPrm SelectPrm var selectPrm SelectPrm
selectPrm.WithContainerID(cnr) selectPrm.WithContainerID(cnr, isIndexedContainer)
selectPrm.WithFilters(fs) selectPrm.WithFilters(fs)
res, err := storage.Select(ctx, selectPrm) res, err := storage.Select(ctx, selectPrm)

View file

@ -50,7 +50,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) {
b.Run("search", func(b *testing.B) { b.Run("search", func(b *testing.B) {
var prm SelectPrm var prm SelectPrm
prm.WithContainerID(cid) prm.WithContainerID(cid, true)
var fs objectSDK.SearchFilters var fs objectSDK.SearchFilters
fs.AddFilter(pilorama.AttributeFilename, strconv.Itoa(objCount/2), objectSDK.MatchStringEqual) fs.AddFilter(pilorama.AttributeFilename, strconv.Itoa(objCount/2), objectSDK.MatchStringEqual)

View file

@ -15,8 +15,9 @@ import (
// SelectPrm groups the parameters of Select operation. // SelectPrm groups the parameters of Select operation.
type SelectPrm struct { type SelectPrm struct {
cnr cid.ID cnr cid.ID
filters objectSDK.SearchFilters filters objectSDK.SearchFilters
isIndexedContainer bool
} }
// SelectRes groups the resulting values of Select operation. // SelectRes groups the resulting values of Select operation.
@ -25,8 +26,9 @@ type SelectRes struct {
} }
// SetContainerID is a Select option to set the container id to search in. // SetContainerID is a Select option to set the container id to search in.
func (p *SelectPrm) SetContainerID(cnr cid.ID) { func (p *SelectPrm) SetContainerID(cnr cid.ID, isIndexedContainer bool) {
p.cnr = cnr p.cnr = cnr
p.isIndexedContainer = isIndexedContainer
} }
// SetFilters is a Select option to set the object filters. // SetFilters is a Select option to set the object filters.
@ -61,6 +63,7 @@ func (s *Shard) Select(ctx context.Context, prm SelectPrm) (SelectRes, error) {
var selectPrm meta.SelectPrm var selectPrm meta.SelectPrm
selectPrm.SetFilters(prm.filters) selectPrm.SetFilters(prm.filters)
selectPrm.SetContainerID(prm.cnr) selectPrm.SetContainerID(prm.cnr)
selectPrm.SetUseAttributeIndex(prm.isIndexedContainer)
mRes, err := s.metaBase.Select(ctx, selectPrm) mRes, err := s.metaBase.Select(ctx, selectPrm)
if err != nil { if err != nil {

View file

@ -8,6 +8,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client"
containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -112,3 +113,12 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error {
return nil return nil
} }
func (exec *execCtx) getContainer() (containerSDK.Container, error) {
cnrID := exec.containerID()
cnr, err := exec.svc.containerSource.Get(cnrID)
if err != nil {
return containerSDK.Container{}, err
}
return cnr.Value, nil
}

View file

@ -54,6 +54,8 @@ type cfg struct {
} }
keyStore *util.KeyStorage keyStore *util.KeyStorage
containerSource container.Source
} }
// New creates, initializes and returns utility serving // New creates, initializes and returns utility serving
@ -63,6 +65,7 @@ func New(e *engine.StorageEngine,
tg *util.TraverserGenerator, tg *util.TraverserGenerator,
ns netmap.Source, ns netmap.Source,
ks *util.KeyStorage, ks *util.KeyStorage,
cs container.Source,
opts ...Option, opts ...Option,
) *Service { ) *Service {
c := &cfg{ c := &cfg{
@ -76,6 +79,7 @@ func New(e *engine.StorageEngine,
traverserGenerator: tg, traverserGenerator: tg,
currentEpochReceiver: ns, currentEpochReceiver: ns,
keyStore: ks, keyStore: ks,
containerSource: cs,
} }
for i := range opts { for i := range opts {

View file

@ -5,6 +5,7 @@ import (
"sync" "sync"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine"
internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util"
@ -112,9 +113,13 @@ func (c *clientWrapper) searchObjects(ctx context.Context, exec *execCtx, info c
} }
func (e *storageEngineWrapper) search(ctx context.Context, exec *execCtx) ([]oid.ID, error) { func (e *storageEngineWrapper) search(ctx context.Context, exec *execCtx) ([]oid.ID, error) {
cnr, err := exec.getContainer()
if err != nil {
return nil, err
}
var selectPrm engine.SelectPrm var selectPrm engine.SelectPrm
selectPrm.WithFilters(exec.searchFilters()) selectPrm.WithFilters(exec.searchFilters())
selectPrm.WithContainerID(exec.containerID()) selectPrm.WithContainerID(exec.containerID(), container.IsIndexedContainer(cnr))
r, err := e.storage.Select(ctx, selectPrm) r, err := e.storage.Select(ctx, selectPrm)
if err != nil { if err != nil {