From 4572fa487493f21cc5ebffcdc526270452a36e6a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 2 Oct 2024 14:52:54 +0300 Subject: [PATCH] [#1412] searchSvc: Check container is indexed For non S3 containers it is expected to use attributes index for some attributes. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 5 +++-- pkg/local_object_storage/engine/delete.go | 2 +- pkg/local_object_storage/engine/inhume_test.go | 4 ++-- pkg/local_object_storage/engine/select.go | 14 ++++++++------ pkg/local_object_storage/engine/tree_test.go | 2 +- pkg/local_object_storage/shard/select.go | 9 ++++++--- pkg/services/object/search/container.go | 10 ++++++++++ pkg/services/object/search/service.go | 4 ++++ pkg/services/object/search/util.go | 7 ++++++- 9 files changed, 41 insertions(+), 16 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index f2c4bff1d..68acb05d3 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -174,7 +174,7 @@ func initObjectService(c *cfg) { sPutV2 := createPutSvcV2(sPut, keyStorage) - sSearch := createSearchSvc(c, keyStorage, traverseGen, c.clientCache) + sSearch := createSearchSvc(c, keyStorage, traverseGen, c.clientCache, c.cfgObject.cnrSource) sSearchV2 := createSearchSvcV2(sSearch, keyStorage) @@ -366,7 +366,7 @@ func createPatchSvc(sGet *getsvc.Service, sPut *putsvc.Service) *patchsvc.Servic return patchsvc.NewService(sPut.Config, sGet) } -func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache) *searchsvc.Service { +func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache, containerSource containercore.Source) *searchsvc.Service { ls := c.cfgObject.cfgLocalStorage.localStorage return searchsvc.New( @@ -377,6 +377,7 @@ func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Trav ), c.netMapSource, keyStorage, + containerSource, searchsvc.WithLogger(c.log), ) } diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 61cb6832d..3ec3f8f9b 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -143,7 +143,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo var selectPrm shard.SelectPrm selectPrm.SetFilters(fs) - selectPrm.SetContainerID(addr.Container()) + selectPrm.SetContainerID(addr.Container(), false) // doesn't matter for search by splitID var inhumePrm shard.InhumePrm if force { diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index f87679253..b4fbbd810 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -49,7 +49,7 @@ func TestStorageEngine_Inhume(t *testing.T) { _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) - addrs, err := Select(context.Background(), e, cnr, fs) + addrs, err := Select(context.Background(), e, cnr, false, fs) require.NoError(t, err) require.Empty(t, addrs) }) @@ -78,7 +78,7 @@ func TestStorageEngine_Inhume(t *testing.T) { _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) - addrs, err := Select(context.Background(), e, cnr, fs) + addrs, err := Select(context.Background(), e, cnr, false, fs) require.NoError(t, err) require.Empty(t, addrs) }) diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 972a4f52a..a85891f0c 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -14,8 +14,9 @@ import ( // SelectPrm groups the parameters of Select operation. type SelectPrm struct { - cnr cid.ID - filters objectSDK.SearchFilters + cnr cid.ID + filters objectSDK.SearchFilters + indexedContainer bool } // SelectRes groups the resulting values of Select operation. @@ -24,8 +25,9 @@ type SelectRes struct { } // WithContainerID is a Select option to set the container id to search in. -func (p *SelectPrm) WithContainerID(cnr cid.ID) { +func (p *SelectPrm) WithContainerID(cnr cid.ID, indexedContainer bool) { p.cnr = cnr + p.indexedContainer = indexedContainer } // WithFilters is a Select option to set the object filters. @@ -67,7 +69,7 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, var outError error var shPrm shard.SelectPrm - shPrm.SetContainerID(prm.cnr) + shPrm.SetContainerID(prm.cnr, prm.indexedContainer) shPrm.SetFilters(prm.filters) e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { @@ -140,9 +142,9 @@ func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, erro } // Select selects objects from local storage using provided filters. -func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs objectSDK.SearchFilters) ([]oid.Address, error) { +func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, isIndexedContainer bool, fs objectSDK.SearchFilters) ([]oid.Address, error) { var selectPrm SelectPrm - selectPrm.WithContainerID(cnr) + selectPrm.WithContainerID(cnr, isIndexedContainer) selectPrm.WithFilters(fs) res, err := storage.Select(ctx, selectPrm) diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index 21fcce415..ea0a9e74e 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -50,7 +50,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { b.Run("search", func(b *testing.B) { var prm SelectPrm - prm.WithContainerID(cid) + prm.WithContainerID(cid, true) var fs objectSDK.SearchFilters fs.AddFilter(pilorama.AttributeFilename, strconv.Itoa(objCount/2), objectSDK.MatchStringEqual) diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go index 1615f5fbe..184ca9b71 100644 --- a/pkg/local_object_storage/shard/select.go +++ b/pkg/local_object_storage/shard/select.go @@ -15,8 +15,9 @@ import ( // SelectPrm groups the parameters of Select operation. type SelectPrm struct { - cnr cid.ID - filters objectSDK.SearchFilters + cnr cid.ID + filters objectSDK.SearchFilters + isIndexedContainer bool } // SelectRes groups the resulting values of Select operation. @@ -25,8 +26,9 @@ type SelectRes struct { } // SetContainerID is a Select option to set the container id to search in. -func (p *SelectPrm) SetContainerID(cnr cid.ID) { +func (p *SelectPrm) SetContainerID(cnr cid.ID, isIndexedContainer bool) { p.cnr = cnr + p.isIndexedContainer = isIndexedContainer } // SetFilters is a Select option to set the object filters. @@ -61,6 +63,7 @@ func (s *Shard) Select(ctx context.Context, prm SelectPrm) (SelectRes, error) { var selectPrm meta.SelectPrm selectPrm.SetFilters(prm.filters) selectPrm.SetContainerID(prm.cnr) + selectPrm.SetUseAttributeIndex(prm.isIndexedContainer) mRes, err := s.metaBase.Select(ctx, selectPrm) if err != nil { diff --git a/pkg/services/object/search/container.go b/pkg/services/object/search/container.go index d70574156..39259b0ca 100644 --- a/pkg/services/object/search/container.go +++ b/pkg/services/object/search/container.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "go.uber.org/zap" ) @@ -112,3 +113,12 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { return nil } + +func (exec *execCtx) getContainer() (containerSDK.Container, error) { + cnrID := exec.containerID() + cnr, err := exec.svc.containerSource.Get(cnrID) + if err != nil { + return containerSDK.Container{}, err + } + return cnr.Value, nil +} diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index cc388c1b2..7700f78d8 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -54,6 +54,8 @@ type cfg struct { } keyStore *util.KeyStorage + + containerSource container.Source } // New creates, initializes and returns utility serving @@ -63,6 +65,7 @@ func New(e *engine.StorageEngine, tg *util.TraverserGenerator, ns netmap.Source, ks *util.KeyStorage, + cs container.Source, opts ...Option, ) *Service { c := &cfg{ @@ -76,6 +79,7 @@ func New(e *engine.StorageEngine, traverserGenerator: tg, currentEpochReceiver: ns, keyStore: ks, + containerSource: cs, } for i := range opts { diff --git a/pkg/services/object/search/util.go b/pkg/services/object/search/util.go index 67b6c0d01..910384a0b 100644 --- a/pkg/services/object/search/util.go +++ b/pkg/services/object/search/util.go @@ -5,6 +5,7 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -112,9 +113,13 @@ func (c *clientWrapper) searchObjects(ctx context.Context, exec *execCtx, info c } func (e *storageEngineWrapper) search(ctx context.Context, exec *execCtx) ([]oid.ID, error) { + cnr, err := exec.getContainer() + if err != nil { + return nil, err + } var selectPrm engine.SelectPrm selectPrm.WithFilters(exec.searchFilters()) - selectPrm.WithContainerID(exec.containerID()) + selectPrm.WithContainerID(exec.containerID(), container.IsIndexedContainer(cnr)) r, err := e.storage.Select(ctx, selectPrm) if err != nil {