TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 110 Pull requests 9 Projects Releases 2 Activity Actions

[#1642] tree: Introduce Cursor type

Browse source 
* Use `Cursor` as parameter for `TreeSortedByFilename`

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
This commit is contained in:
Airat Arifullin 2025-02-20 11:05:11 +03:00 • committed by Evgenii Stratonikov
parent a405fb1f39
commit a11b2d27e4
9 changed files with 40 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pkg/local_object_storage/engine/tree.go
View file

@ -230,7 +230,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree
}
// TreeSortedByFilename implements the pilorama.Forest interface.
func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *string, count int) ([]pilorama.MultiNodeInfo, *string, error) {
func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *pilorama.Cursor, count int) ([]pilorama.MultiNodeInfo, *pilorama.Cursor, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeSortedByFilename",
trace.WithAttributes(
attribute.String("container_id", cid.EncodeToString()),
@ -241,7 +241,7 @@ func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID,
var err error
var nodes []pilorama.MultiNodeInfo
var cursor *string
var cursor *pilorama.Cursor
for _, sh := range e.sortShards(cid) {
nodes, cursor, err = sh.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count)
if err != nil {

9
pkg/local_object_storage/pilorama/boltdb.go
View file

@ -1077,7 +1077,7 @@ func (t *boltForest) hasFewChildren(b *bbolt.Bucket, nodeIDs MultiNode, threshol
}
// TreeSortedByFilename implements the Forest interface.
func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeIDs MultiNode, last *string, count int) ([]MultiNodeInfo, *string, error) {
func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeIDs MultiNode, last *Cursor, count int) ([]MultiNodeInfo, *Cursor, error) {
var (
startedAt = time.Now()
success = false
@ -1128,7 +1128,6 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr
}
t.fillSortedChildren(b, nodeIDs, h)
for info, ok := h.pop(); ok; info, ok = h.pop() {
for _, id := range info.id {
childInfo, err := t.getChildInfo(b, key, id)
@ -1155,7 +1154,7 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr
}
if len(res) != 0 {
s := string(findAttr(res[len(res)-1].Meta, AttributeFilename))
last = &s
last = NewCursor(s)
}
return res, last, metaerr.Wrap(err)
}
@ -1166,10 +1165,10 @@ func sortByFilename(nodes []NodeInfo) {
})
}
func sortAndCut(result []NodeInfo, last *string) []NodeInfo {
func sortAndCut(result []NodeInfo, last *Cursor) []NodeInfo {
var lastBytes []byte
if last != nil {
lastBytes = []byte(*last)
lastBytes = []byte(last.GetFilename())
}
sortByFilename(result)

8
pkg/local_object_storage/pilorama/forest.go
View file

@ -164,7 +164,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string,
}
// TreeSortedByFilename implements the Forest interface.
func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeIDs MultiNode, start *string, count int) ([]MultiNodeInfo, *string, error) {
func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeIDs MultiNode, start *Cursor, count int) ([]MultiNodeInfo, *Cursor, error) {
fullID := cid.String() + "/" + treeID
s, ok := f.treeMap[fullID]
if !ok {
@ -204,14 +204,14 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI
r := mergeNodeInfos(res)
for i := range r {
if start == nil || string(findAttr(r[i].Meta, AttributeFilename)) > *start {
if start == nil || string(findAttr(r[i].Meta, AttributeFilename)) > start.GetFilename() {
finish := min(len(res), i+count)
last := string(findAttr(r[finish-1].Meta, AttributeFilename))
return r[i:finish], &last, nil
return r[i:finish], NewCursor(last), nil
}
}
last := string(res[len(res)-1].Meta.GetAttr(AttributeFilename))
return nil, &last, nil
return nil, NewCursor(last), nil
}
// TreeGetChildren implements the Forest interface.

4
pkg/local_object_storage/pilorama/forest_test.go
View file

@ -273,7 +273,7 @@ func testForestTreeSortedIterationBugWithSkip(t *testing.T, s ForestStorage) {
}
var result []MultiNodeInfo
treeAppend := func(t *testing.T, last *string, count int) *string {
treeAppend := func(t *testing.T, last *Cursor, count int) *Cursor {
res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, last, count)
require.NoError(t, err)
result = append(result, res...)
@ -328,7 +328,7 @@ func testForestTreeSortedIteration(t *testing.T, s ForestStorage) {
}
var result []MultiNodeInfo
treeAppend := func(t *testing.T, last *string, count int) *string {
treeAppend := func(t *testing.T, last *Cursor, count int) *Cursor {
res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, last, count)
require.NoError(t, err)
result = append(result, res...)

6
pkg/local_object_storage/pilorama/heap.go
View file

@ -30,13 +30,13 @@ func (h *filenameHeap) Pop() any {
// fixedHeap maintains a fixed number of smallest elements started at some point.
type fixedHeap struct {
start *string
start *Cursor
sorted bool
count int
h *filenameHeap
}
func newHeap(start *string, count int) *fixedHeap {
func newHeap(start *Cursor, count int) *fixedHeap {
h := new(filenameHeap)
heap.Init(h)
@ -50,7 +50,7 @@ func newHeap(start *string, count int) *fixedHeap {
const amortizationMultiplier = 5
func (h *fixedHeap) push(id MultiNode, filename string) bool {
if h.start != nil && filename <= *h.start {
if h.start != nil && filename <= (*h.start).GetFilename() {
return false
}

23
pkg/local_object_storage/pilorama/interface.go
View file

@ -37,7 +37,7 @@ type Forest interface {
TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error)
// TreeSortedByFilename returns children of the node with the specified ID. The nodes are sorted by the filename attribute..
// Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree.
TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID MultiNode, last *string, count int) ([]MultiNodeInfo, *string, error)
TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID MultiNode, last *Cursor, count int) ([]MultiNodeInfo, *Cursor, error)
// TreeGetOpLog returns first log operation stored at or above the height.
// In case no such operation is found, empty Move and nil error should be returned.
TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error)
@ -79,6 +79,27 @@ const (
AttributeVersion = "Version"
)
// Cursor keeps state between function calls for traversing nodes.
// It stores the attributes associated with a previous call, allowing subsequent operations
// to resume traversal from this point rather than starting from the beginning.
type Cursor struct {
// Last traversed filename.
filename string
}
func NewCursor(filename string) *Cursor {
return &Cursor{
filename: filename,
}
}
func (c *Cursor) GetFilename() string {
if c == nil {
return ""
}
return c.filename
}
// CIDDescriptor contains container ID and information about the node position
// in the list of container nodes.
type CIDDescriptor struct {

2
pkg/local_object_storage/pilorama/split_test.go
View file

@ -96,7 +96,7 @@ func testDuplicateDirectory(t *testing.T, f Forest) {
require.Equal(t, []byte{8}, testGetByPath(t, "dir1/dir3/value4"))
require.Equal(t, []byte{10}, testGetByPath(t, "value0"))
testSortedByFilename := func(t *testing.T, root MultiNode, last *string, batchSize int) ([]MultiNodeInfo, *string) {
testSortedByFilename := func(t *testing.T, root MultiNode, last *Cursor, batchSize int) ([]MultiNodeInfo, *Cursor) {
res, last, err := f.TreeSortedByFilename(context.Background(), d.CID, treeID, root, last, batchSize)
require.NoError(t, err)
return res, last

2
pkg/local_object_storage/shard/tree.go
View file

@ -246,7 +246,7 @@ func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID strin
}
// TreeSortedByFilename implements the pilorama.Forest interface.
func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *string, count int) ([]pilorama.MultiNodeInfo, *string, error) {
func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *pilorama.Cursor, count int) ([]pilorama.MultiNodeInfo, *pilorama.Cursor, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeSortedByFilename",
trace.WithAttributes(
attribute.String("shard_id", s.ID().String()),

2
pkg/services/tree/service.go
View file

@ -412,7 +412,7 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS
type stackItem struct {
values []pilorama.MultiNodeInfo
parent pilorama.MultiNode
last *string
last *pilorama.Cursor
}
func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error {