From 669571f367cde380bdb74298f18cad508cff8b3f Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 20 Feb 2025 11:05:11 +0300 Subject: [PATCH] [#1642] tree: Rename `TreeSortedByFilename` method's parameter Signed-off-by: Airat Arifullin --- pkg/local_object_storage/engine/tree.go | 7 +++--- pkg/local_object_storage/pilorama/boltdb.go | 22 +++++++++---------- pkg/local_object_storage/pilorama/forest.go | 8 +++---- .../pilorama/forest_test.go | 8 +++---- pkg/local_object_storage/pilorama/heap.go | 6 ++--- .../pilorama/interface.go | 10 ++++++++- .../pilorama/split_test.go | 6 ++--- pkg/local_object_storage/shard/tree.go | 8 +++---- pkg/services/tree/service.go | 10 ++++----- 9 files changed, 46 insertions(+), 39 deletions(-) diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 7f70d36f7..ed7643976 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -230,7 +230,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree } // TreeSortedByFilename implements the pilorama.Forest interface. -func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *string, count int) ([]pilorama.MultiNodeInfo, *string, error) { +func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, cursor *pilorama.Cursor, count int) ([]pilorama.MultiNodeInfo, *pilorama.Cursor, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeSortedByFilename", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -241,9 +241,8 @@ func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, var err error var nodes []pilorama.MultiNodeInfo - var cursor *string for _, sh := range e.sortShards(cid) { - nodes, cursor, err = sh.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count) + nodes, cursor, err = sh.TreeSortedByFilename(ctx, cid, treeID, nodeID, cursor, count) if err != nil { if err == shard.ErrPiloramaDisabled { break @@ -257,7 +256,7 @@ func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, } return nodes, cursor, nil } - return nil, last, err + return nil, cursor, err } // TreeGetOpLog implements the pilorama.Forest interface. diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 9d71d9fda..901bfd58e 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1077,7 +1077,7 @@ func (t *boltForest) hasFewChildren(b *bbolt.Bucket, nodeIDs MultiNode, threshol } // TreeSortedByFilename implements the Forest interface. -func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeIDs MultiNode, last *string, count int) ([]MultiNodeInfo, *string, error) { +func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeIDs MultiNode, cursor *Cursor, count int) ([]MultiNodeInfo, *Cursor, error) { var ( startedAt = time.Now() success = false @@ -1098,13 +1098,13 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr defer t.modeMtx.RUnlock() if t.mode.NoMetabase() { - return nil, last, ErrDegradedMode + return nil, cursor, ErrDegradedMode } if len(nodeIDs) == 0 { - return nil, last, errors.New("empty node list") + return nil, cursor, errors.New("empty node list") } - h := newHeap(last, count) + h := newHeap(cursor, count) key := make([]byte, 9) var result []NodeInfo @@ -1143,11 +1143,11 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr success = err == nil if err != nil { - return nil, last, metaerr.Wrap(err) + return nil, cursor, metaerr.Wrap(err) } if fewChildren { - result = sortAndCut(result, last) + result = sortAndCut(result, cursor) } res := mergeNodeInfos(result) if len(res) > count { @@ -1155,9 +1155,9 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr } if len(res) != 0 { s := string(findAttr(res[len(res)-1].Meta, AttributeFilename)) - last = &s + cursor = &Cursor{Filename: s} } - return res, last, metaerr.Wrap(err) + return res, cursor, metaerr.Wrap(err) } func sortByFilename(nodes []NodeInfo) { @@ -1166,10 +1166,10 @@ func sortByFilename(nodes []NodeInfo) { }) } -func sortAndCut(result []NodeInfo, last *string) []NodeInfo { +func sortAndCut(result []NodeInfo, cursor *Cursor) []NodeInfo { var lastBytes []byte - if last != nil { - lastBytes = []byte(*last) + if cursor != nil { + lastBytes = []byte((*cursor).Filename) } sortByFilename(result) diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 92183716c..7a4212f79 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -164,7 +164,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string, } // TreeSortedByFilename implements the Forest interface. -func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeIDs MultiNode, start *string, count int) ([]MultiNodeInfo, *string, error) { +func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeIDs MultiNode, start *Cursor, count int) ([]MultiNodeInfo, *Cursor, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -204,17 +204,17 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI r := mergeNodeInfos(res) for i := range r { - if start == nil || string(findAttr(r[i].Meta, AttributeFilename)) > *start { + if start == nil || string(findAttr(r[i].Meta, AttributeFilename)) > (*start).Filename { finish := i + count if len(res) < finish { finish = len(res) } last := string(findAttr(r[finish-1].Meta, AttributeFilename)) - return r[i:finish], &last, nil + return r[i:finish], &Cursor{Filename: last}, nil } } last := string(res[len(res)-1].Meta.GetAttr(AttributeFilename)) - return nil, &last, nil + return nil, &Cursor{Filename: last}, nil } // TreeGetChildren implements the Forest interface. diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index de56fc82b..1bf710927 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -273,8 +273,8 @@ func testForestTreeSortedIterationBugWithSkip(t *testing.T, s ForestStorage) { } var result []MultiNodeInfo - treeAppend := func(t *testing.T, last *string, count int) *string { - res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, last, count) + treeAppend := func(t *testing.T, cursor *Cursor, count int) *Cursor { + res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, cursor, count) require.NoError(t, err) result = append(result, res...) return cursor @@ -328,8 +328,8 @@ func testForestTreeSortedIteration(t *testing.T, s ForestStorage) { } var result []MultiNodeInfo - treeAppend := func(t *testing.T, last *string, count int) *string { - res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, last, count) + treeAppend := func(t *testing.T, cursor *Cursor, count int) *Cursor { + res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, cursor, count) require.NoError(t, err) result = append(result, res...) return cursor diff --git a/pkg/local_object_storage/pilorama/heap.go b/pkg/local_object_storage/pilorama/heap.go index 5a00bcf7a..c59a02b09 100644 --- a/pkg/local_object_storage/pilorama/heap.go +++ b/pkg/local_object_storage/pilorama/heap.go @@ -30,13 +30,13 @@ func (h *filenameHeap) Pop() any { // fixedHeap maintains a fixed number of smallest elements started at some point. type fixedHeap struct { - start *string + start *Cursor sorted bool count int h *filenameHeap } -func newHeap(start *string, count int) *fixedHeap { +func newHeap(start *Cursor, count int) *fixedHeap { h := new(filenameHeap) heap.Init(h) @@ -50,7 +50,7 @@ func newHeap(start *string, count int) *fixedHeap { const amortizationMultiplier = 5 func (h *fixedHeap) push(id MultiNode, filename string) bool { - if h.start != nil && filename <= *h.start { + if h.start != nil && filename <= (*h.start).Filename { return false } diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 1f7e742a2..b7694ab7c 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -37,7 +37,7 @@ type Forest interface { TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) // TreeSortedByFilename returns children of the node with the specified ID. The nodes are sorted by the filename attribute.. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. - TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID MultiNode, last *string, count int) ([]MultiNodeInfo, *string, error) + TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID MultiNode, cursor *Cursor, count int) ([]MultiNodeInfo, *Cursor, error) // TreeGetOpLog returns first log operation stored at or above the height. // In case no such operation is found, empty Move and nil error should be returned. TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) @@ -79,6 +79,14 @@ const ( AttributeVersion = "Version" ) +// Cursor keeps state between function calls for traversing nodes. +// It stores the attributes associated with a previous call, allowing subsequent operations +// to resume traversal from this point rather than starting from the beginning. +type Cursor struct { + // Last traversed filename. + Filename string +} + // CIDDescriptor contains container ID and information about the node position // in the list of container nodes. type CIDDescriptor struct { diff --git a/pkg/local_object_storage/pilorama/split_test.go b/pkg/local_object_storage/pilorama/split_test.go index 54c2b90a6..962814170 100644 --- a/pkg/local_object_storage/pilorama/split_test.go +++ b/pkg/local_object_storage/pilorama/split_test.go @@ -96,10 +96,10 @@ func testDuplicateDirectory(t *testing.T, f Forest) { require.Equal(t, []byte{8}, testGetByPath(t, "dir1/dir3/value4")) require.Equal(t, []byte{10}, testGetByPath(t, "value0")) - testSortedByFilename := func(t *testing.T, root MultiNode, last *string, batchSize int) ([]MultiNodeInfo, *string) { - res, last, err := f.TreeSortedByFilename(context.Background(), d.CID, treeID, root, last, batchSize) + testSortedByFilename := func(t *testing.T, root MultiNode, cursor *Cursor, batchSize int) ([]MultiNodeInfo, *Cursor) { + res, cursor, err := f.TreeSortedByFilename(context.Background(), d.CID, treeID, root, cursor, batchSize) require.NoError(t, err) - return res, last + return res, cursor } t.Run("test sorted listing, full children branch", func(t *testing.T) { diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 01a014cec..47a3e4c34 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -211,7 +211,7 @@ func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID strin } // TreeSortedByFilename implements the pilorama.Forest interface. -func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *string, count int) ([]pilorama.MultiNodeInfo, *string, error) { +func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, cursor *pilorama.Cursor, count int) ([]pilorama.MultiNodeInfo, *pilorama.Cursor, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeSortedByFilename", trace.WithAttributes( attribute.String("shard_id", s.ID().String()), @@ -222,16 +222,16 @@ func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID defer span.End() if s.pilorama == nil { - return nil, last, ErrPiloramaDisabled + return nil, cursor, ErrPiloramaDisabled } s.m.RLock() defer s.m.RUnlock() if s.info.Mode.NoMetabase() { - return nil, last, ErrDegradedMode + return nil, cursor, ErrDegradedMode } - return s.pilorama.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count) + return s.pilorama.TreeSortedByFilename(ctx, cid, treeID, nodeID, cursor, count) } // TreeGetOpLog implements the pilorama.Forest interface. diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 2e9722e79..ef0a38bc1 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -409,7 +409,7 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS type stackItem struct { values []pilorama.MultiNodeInfo parent pilorama.MultiNode - last *string + cursor *pilorama.Cursor } func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { @@ -460,12 +460,12 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid break } - nodes, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), item.parent, item.last, batchSize) + nodes, cursor, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), item.parent, item.cursor, batchSize) if err != nil { return err } item.values = nodes - item.last = last + item.cursor = cursor if len(nodes) == 0 { stack = stack[:len(stack)-1] @@ -479,7 +479,7 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid } if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() { - children, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), node.Children, nil, batchSize) + children, cursor, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), node.Children, nil, batchSize) if err != nil { return err } @@ -487,7 +487,7 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid stack = append(stack, stackItem{ values: children, parent: node.Children, - last: last, + cursor: cursor, }) } }