aarifullin/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#1642] tree: Rename TreeSortedByFilename method's parameter

Browse source 
Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
This commit is contained in:
Airat Arifullin 2025-02-20 11:05:11 +03:00
parent 9a0507704a
commit 669571f367
9 changed files with 46 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
pkg/local_object_storage/engine/tree.go
View file

@ -230,7 +230,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree
}
// TreeSortedByFilename implements the pilorama.Forest interface.
func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *string, count int) ([]pilorama.MultiNodeInfo, *string, error) {
func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, cursor *pilorama.Cursor, count int) ([]pilorama.MultiNodeInfo, *pilorama.Cursor, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeSortedByFilename",
trace.WithAttributes(
attribute.String("container_id", cid.EncodeToString()),
@ -241,9 +241,8 @@ func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID,
var err error
var nodes []pilorama.MultiNodeInfo
var cursor *string
for _, sh := range e.sortShards(cid) {
nodes, cursor, err = sh.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count)
nodes, cursor, err = sh.TreeSortedByFilename(ctx, cid, treeID, nodeID, cursor, count)
if err != nil {
if err == shard.ErrPiloramaDisabled {
break
@ -257,7 +256,7 @@ func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID,
}
return nodes, cursor, nil
}
return nil, last, err
return nil, cursor, err
}
// TreeGetOpLog implements the pilorama.Forest interface.

22
pkg/local_object_storage/pilorama/boltdb.go
View file

@ -1077,7 +1077,7 @@ func (t *boltForest) hasFewChildren(b *bbolt.Bucket, nodeIDs MultiNode, threshol
}
// TreeSortedByFilename implements the Forest interface.
func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeIDs MultiNode, last *string, count int) ([]MultiNodeInfo, *string, error) {
func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeIDs MultiNode, cursor *Cursor, count int) ([]MultiNodeInfo, *Cursor, error) {
var (
startedAt = time.Now()
success = false
@ -1098,13 +1098,13 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr
defer t.modeMtx.RUnlock()
if t.mode.NoMetabase() {
return nil, last, ErrDegradedMode
return nil, cursor, ErrDegradedMode
}
if len(nodeIDs) == 0 {
return nil, last, errors.New("empty node list")
return nil, cursor, errors.New("empty node list")
}
h := newHeap(last, count)
h := newHeap(cursor, count)
key := make([]byte, 9)
var result []NodeInfo
@ -1143,11 +1143,11 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr
success = err == nil
if err != nil {
return nil, last, metaerr.Wrap(err)
return nil, cursor, metaerr.Wrap(err)
}
if fewChildren {
result = sortAndCut(result, last)
result = sortAndCut(result, cursor)
}
res := mergeNodeInfos(result)
if len(res) > count {
@ -1155,9 +1155,9 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr
}
if len(res) != 0 {
s := string(findAttr(res[len(res)-1].Meta, AttributeFilename))
last = &s
cursor = &Cursor{Filename: s}
}
return res, last, metaerr.Wrap(err)
return res, cursor, metaerr.Wrap(err)
}
func sortByFilename(nodes []NodeInfo) {
@ -1166,10 +1166,10 @@ func sortByFilename(nodes []NodeInfo) {
})
}
func sortAndCut(result []NodeInfo, last *string) []NodeInfo {
func sortAndCut(result []NodeInfo, cursor *Cursor) []NodeInfo {
var lastBytes []byte
if last != nil {
lastBytes = []byte(*last)
if cursor != nil {
lastBytes = []byte((*cursor).Filename)
}
sortByFilename(result)

8
pkg/local_object_storage/pilorama/forest.go
View file

@ -164,7 +164,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string,
}
// TreeSortedByFilename implements the Forest interface.
func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeIDs MultiNode, start *string, count int) ([]MultiNodeInfo, *string, error) {
func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeIDs MultiNode, start *Cursor, count int) ([]MultiNodeInfo, *Cursor, error) {
fullID := cid.String() + "/" + treeID
s, ok := f.treeMap[fullID]
if !ok {
@ -204,17 +204,17 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI
r := mergeNodeInfos(res)
for i := range r {
if start == nil || string(findAttr(r[i].Meta, AttributeFilename)) > *start {
if start == nil || string(findAttr(r[i].Meta, AttributeFilename)) > (*start).Filename {
finish := i + count
if len(res) < finish {
finish = len(res)
}
last := string(findAttr(r[finish-1].Meta, AttributeFilename))
return r[i:finish], &last, nil
return r[i:finish], &Cursor{Filename: last}, nil
}
}
last := string(res[len(res)-1].Meta.GetAttr(AttributeFilename))
return nil, &last, nil
return nil, &Cursor{Filename: last}, nil
}
// TreeGetChildren implements the Forest interface.

8
pkg/local_object_storage/pilorama/forest_test.go
View file

@ -273,8 +273,8 @@ func testForestTreeSortedIterationBugWithSkip(t *testing.T, s ForestStorage) {
}
var result []MultiNodeInfo
treeAppend := func(t *testing.T, last *string, count int) *string {
res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, last, count)
treeAppend := func(t *testing.T, cursor *Cursor, count int) *Cursor {
res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, cursor, count)
require.NoError(t, err)
result = append(result, res...)
return cursor
@ -328,8 +328,8 @@ func testForestTreeSortedIteration(t *testing.T, s ForestStorage) {
}
var result []MultiNodeInfo
treeAppend := func(t *testing.T, last *string, count int) *string {
res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, last, count)
treeAppend := func(t *testing.T, cursor *Cursor, count int) *Cursor {
res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, cursor, count)
require.NoError(t, err)
result = append(result, res...)
return cursor

6
pkg/local_object_storage/pilorama/heap.go
View file

@ -30,13 +30,13 @@ func (h *filenameHeap) Pop() any {
// fixedHeap maintains a fixed number of smallest elements started at some point.
type fixedHeap struct {
start *string
start *Cursor
sorted bool
count int
h *filenameHeap
}
func newHeap(start *string, count int) *fixedHeap {
func newHeap(start *Cursor, count int) *fixedHeap {
h := new(filenameHeap)
heap.Init(h)
@ -50,7 +50,7 @@ func newHeap(start *string, count int) *fixedHeap {
const amortizationMultiplier = 5
func (h *fixedHeap) push(id MultiNode, filename string) bool {
if h.start != nil && filename <= *h.start {
if h.start != nil && filename <= (*h.start).Filename {
return false
}

10
pkg/local_object_storage/pilorama/interface.go
View file

@ -37,7 +37,7 @@ type Forest interface {
TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error)
// TreeSortedByFilename returns children of the node with the specified ID. The nodes are sorted by the filename attribute..
// Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree.
TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID MultiNode, last *string, count int) ([]MultiNodeInfo, *string, error)
TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID MultiNode, cursor *Cursor, count int) ([]MultiNodeInfo, *Cursor, error)
// TreeGetOpLog returns first log operation stored at or above the height.
// In case no such operation is found, empty Move and nil error should be returned.
TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error)
@ -79,6 +79,14 @@ const (
AttributeVersion = "Version"
)
// Cursor keeps state between function calls for traversing nodes.
// It stores the attributes associated with a previous call, allowing subsequent operations
// to resume traversal from this point rather than starting from the beginning.
type Cursor struct {
// Last traversed filename.
Filename string
}
// CIDDescriptor contains container ID and information about the node position
// in the list of container nodes.
type CIDDescriptor struct {

6
pkg/local_object_storage/pilorama/split_test.go
View file

@ -96,10 +96,10 @@ func testDuplicateDirectory(t *testing.T, f Forest) {
require.Equal(t, []byte{8}, testGetByPath(t, "dir1/dir3/value4"))
require.Equal(t, []byte{10}, testGetByPath(t, "value0"))
testSortedByFilename := func(t *testing.T, root MultiNode, last *string, batchSize int) ([]MultiNodeInfo, *string) {
res, last, err := f.TreeSortedByFilename(context.Background(), d.CID, treeID, root, last, batchSize)
testSortedByFilename := func(t *testing.T, root MultiNode, cursor *Cursor, batchSize int) ([]MultiNodeInfo, *Cursor) {
res, cursor, err := f.TreeSortedByFilename(context.Background(), d.CID, treeID, root, cursor, batchSize)
require.NoError(t, err)
return res, last
return res, cursor
}
t.Run("test sorted listing, full children branch", func(t *testing.T) {

8
pkg/local_object_storage/shard/tree.go
View file

@ -211,7 +211,7 @@ func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID strin
}
// TreeSortedByFilename implements the pilorama.Forest interface.
func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *string, count int) ([]pilorama.MultiNodeInfo, *string, error) {
func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, cursor *pilorama.Cursor, count int) ([]pilorama.MultiNodeInfo, *pilorama.Cursor, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeSortedByFilename",
trace.WithAttributes(
attribute.String("shard_id", s.ID().String()),
@ -222,16 +222,16 @@ func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID
defer span.End()
if s.pilorama == nil {
return nil, last, ErrPiloramaDisabled
return nil, cursor, ErrPiloramaDisabled
}
s.m.RLock()
defer s.m.RUnlock()
if s.info.Mode.NoMetabase() {
return nil, last, ErrDegradedMode
return nil, cursor, ErrDegradedMode
}
return s.pilorama.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count)
return s.pilorama.TreeSortedByFilename(ctx, cid, treeID, nodeID, cursor, count)
}
// TreeGetOpLog implements the pilorama.Forest interface.

10
pkg/services/tree/service.go
View file

@ -409,7 +409,7 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS
type stackItem struct {
values []pilorama.MultiNodeInfo
parent pilorama.MultiNode
last *string
cursor *pilorama.Cursor
}
func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error {
@ -460,12 +460,12 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid
break
}
nodes, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), item.parent, item.last, batchSize)
nodes, cursor, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), item.parent, item.cursor, batchSize)
if err != nil {
return err
}
item.values = nodes
item.last = last
item.cursor = cursor
if len(nodes) == 0 {
stack = stack[:len(stack)-1]
@ -479,7 +479,7 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid
}
if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() {
children, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), node.Children, nil, batchSize)
children, cursor, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), node.Children, nil, batchSize)
if err != nil {
return err
}
@ -487,7 +487,7 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid
stack = append(stack, stackItem{
values: children,
parent: node.Children,
last: last,
cursor: cursor,
})
}
}