m

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>

pkg/local_object_storage/pilorama/boltdb.go

@@ -1168,13 +1168,13 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr
 }
 
 type ListKeysCursor struct {
-	prefix   string
+	prefix   []string
 	finished bool
 	stack    stack
 }
 
 func (c *ListKeysCursor) SetPrefix(prefix string) {
-	c.prefix = prefix
+	c.prefix = strings.Split(prefix, "/")
 }
 
 func (t *boltForest) TreeListKeys(_ context.Context, cid cidSDK.ID, treeID string, cursor ListKeysCursor, count int) ([]KeyInfo, ListKeysCursor, error) {
@@ -1198,7 +1198,7 @@ func (t *boltForest) TreeListKeys(_ context.Context, cid cidSDK.ID, treeID strin
 
 		b := treeRoot.Bucket(dataBucket)
 		var err error
-		result, err = t.listKeys(b, &cursor, count, cursor.prefix, MultiNode{RootID})
+		result, err = t.listKeys(b, &cursor, count, MultiNode{RootID})
 		return err
 	})
 
@@ -1208,7 +1208,7 @@ func (t *boltForest) TreeListKeys(_ context.Context, cid cidSDK.ID, treeID strin
 	return result, cursor, metaerr.Wrap(err)
 }
 
-func (t *boltForest) listKeys(b *bbolt.Bucket, cursor *ListKeysCursor, count int, filterPrefix string, root MultiNode) ([]KeyInfo, error) {
+func (t *boltForest) listKeys(b *bbolt.Bucket, cursor *ListKeysCursor, count int, root MultiNode) ([]KeyInfo, error) {
 	var result []KeyInfo
 	if cursor.stack.empty() {
 		cursor.stack.push(treeLevel{
@@ -1232,11 +1232,18 @@ func (t *boltForest) listKeys(b *bbolt.Bucket, cursor *ListKeysCursor, count int
 	// }
 
 	for !cursor.stack.empty() {
+		depth := len(cursor.stack.values)
 		node, _, prefix := cursor.stack.popSingle()
 		if strings.HasSuffix(node.filename, "/") {
 			h := newHeap(nil, count*5)
 			h.withSlash = true
-			t.fillSortedChildren(b, node.id, h)
+			if depth < len(cursor.prefix) {
+				t.fillSortedChildrenExact(b, node.id, h, cursor.prefix[depth-1])
+			} else if depth == len(cursor.prefix) {
+				t.fillSortedChildrenWithPrefix(b, node.id, h, cursor.prefix[depth-1])
+			} else {
+				t.fillSortedChildren(b, node.id, h)
+			}
 
 			var children []heapInfo
 			for hi, ok := h.pop(); ok; hi, ok = h.pop() {
@@ -1249,12 +1256,9 @@ func (t *boltForest) listKeys(b *bbolt.Bucket, cursor *ListKeysCursor, count int
 					children:     children,
 				})
 			}
-		} else {
+			cursor.stack.trim()
+		} else if len(cursor.prefix) < depth {
 			treeKey := strings.TrimPrefix(prefix, "/") + node.filename
-			if !strings.HasPrefix(treeKey, filterPrefix) {
-				continue
-			}
-
 			meta, err := t.getLatestMeta(b, node.id)
 			if err != nil {
 				return nil, err
@@ -1396,6 +1400,102 @@ func (t *boltForest) getChildInfo(b *bbolt.Bucket, key []byte, childID Node) (No
 	return childInfo, nil
 }
 
+func (t *boltForest) fillSortedChildrenExact(b *bbolt.Bucket, nodeIDs MultiNode, h *fixedHeap, pp string) {
+	c := b.Cursor()
+
+	length := uint16(len(pp))
+	prefix := internalKeyPrefix(nil, AttributeFilename)
+	prefix = binary.LittleEndian.AppendUint16(prefix, length)
+	prefix = append(prefix, pp...)
+
+	var nodes []uint64
+	for k, _ := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, _ = c.Next() {
+		parentID := binary.LittleEndian.Uint64(k[len(k)-16:])
+		if !slices.Contains(nodeIDs, parentID) {
+			continue
+		}
+
+		childID := binary.LittleEndian.Uint64(k[len(k)-8:])
+		nodes = append(nodes, childID)
+	}
+
+	if len(nodes) != 0 {
+		h.push(nodes, pp)
+	}
+}
+
+func (t *boltForest) fillSortedChildrenWithPrefix(b *bbolt.Bucket, nodeIDs MultiNode, h *fixedHeap, pp string) {
+	c := b.Cursor()
+
+	length := uint16(len(pp))
+	prefix := internalKeyPrefix(nil, AttributeFilename)
+
+	offset := len(prefix)
+	prefix = binary.LittleEndian.AppendUint16(prefix, length)
+	prefix = append(prefix, pp...)
+
+	count := 0
+
+	var nodes []uint64
+	var lastFilename *string
+	for k, _ := c.Seek(prefix); len(k) > 0 && k[0] == 'i'; k, _ = c.Next() {
+		if len(k) < len(prefix)+16 {
+			continue
+		}
+
+		actualLength := binary.LittleEndian.Uint16(k[offset:])
+		childID := binary.LittleEndian.Uint64(k[len(k)-8:])
+		filename := string(k[offset+2 : len(k)-16])
+		if !strings.HasPrefix(filename, pp) {
+			if lastFilename != nil {
+				h.push(nodes, *lastFilename)
+			}
+			lastFilename = nil
+			nodes = nil
+			length = actualLength + 1
+			count = 0
+			binary.LittleEndian.PutUint16(prefix[offset:], length)
+			c.Seek(prefix)
+			c.Prev() // c.Next() will be performed by for loop
+			continue
+		}
+
+		parentID := binary.LittleEndian.Uint64(k[len(k)-16:])
+		if !slices.Contains(nodeIDs, parentID) {
+			continue
+		}
+
+		if lastFilename == nil {
+			lastFilename = &filename
+			nodes = append(nodes, childID)
+		} else if *lastFilename == filename {
+			nodes = append(nodes, childID)
+		} else {
+			processed := h.push(nodes, *lastFilename)
+			nodes = MultiNode{childID}
+			lastFilename = &filename
+			if actualLength != length {
+				length = actualLength
+				count = 1
+			} else if processed {
+				if count++; count > h.count {
+					lastFilename = nil
+					nodes = nil
+					length = actualLength + 1
+					count = 0
+					binary.LittleEndian.PutUint16(prefix[offset:], length)
+					c.Seek(prefix)
+					c.Prev() // c.Next() will be performed by for loop
+				}
+			}
+		}
+	}
+
+	if len(nodes) != 0 && lastFilename != nil {
+		h.push(nodes, *lastFilename)
+	}
+}
+
 func (t *boltForest) fillSortedChildren(b *bbolt.Bucket, nodeIDs MultiNode, h *fixedHeap) {
 	c := b.Cursor()
 	prefix := internalKeyPrefix(nil, AttributeFilename)

pkg/local_object_storage/pilorama/boltdb_listing_test.go

@@ -8,7 +8,6 @@ import (
 	"testing"
 
 	cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
-	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 )
 
@@ -44,7 +43,7 @@ func testForestTreeListKeys(t *testing.T, s *boltForest) {
 	for i := range keys {
 		path := strings.Split(keys[i].Key, "/")
 		keys[i].Meta = []KeyValue{
-			{Key: AttributeVersion, Value: []byte(uuid.New().String())},
+			{Key: AttributeVersion, Value: []byte{byte(i)}}, // uuid.New().String())},
 			{Key: AttributeFilename, Value: []byte(path[len(path)-1])},
 		}
 		_, err := s.TreeAddByPath(ctx, d, treeID, AttributeFilename, path[:len(path)-1], keys[i].Meta)
@@ -56,14 +55,13 @@ func testForestTreeListKeys(t *testing.T, s *boltForest) {
 		return strings.Compare(a.Key, b.Key)
 	})
 
-	cursor := ListKeysCursor{}
 	listKeys := func(t *testing.T, cursor ListKeysCursor, count int) ([]KeyInfo, ListKeysCursor) {
 		res, cursor, err := s.TreeListKeys(ctx, cid, treeID, cursor, count)
 		require.NoError(t, err)
 		return res, cursor
 	}
 
-	res, _ := listKeys(t, cursor, 20)
+	res, _ := listKeys(t, ListKeysCursor{}, 20)
 	require.Equal(t, sortedKeys, res)
 
 	t.Run("multiple invocations", func(t *testing.T) {

pkg/local_object_storage/pilorama/stack.go

@@ -1,6 +1,8 @@
 package pilorama
 
-import "go.etcd.io/bbolt"
+import (
+	"go.etcd.io/bbolt"
+)
 
 type treeLevel struct {
 	parentPrefix string
@@ -13,7 +15,12 @@ type stack struct {
 }
 
 func (s *stack) empty() bool {
-	return len(s.values) == 0
+	for i := range s.values {
+		if len(s.values[i].children) != 0 {
+			return false
+		}
+	}
+	return true
 }
 
 func (s *stack) popSingle() (heapInfo, MultiNode, string) {
@@ -23,11 +30,13 @@ func (s *stack) popSingle() (heapInfo, MultiNode, string) {
 	head, tail := lastLevel.children[0], lastLevel.children[1:]
 	lastLevel.children = tail
 
+	return head, parents, prefix
+}
+
+func (s *stack) trim() {
 	for len(s.values) > 0 && len(s.values[len(s.values)-1].children) == 0 {
 		s.values = s.values[:len(s.values)-1]
 	}
-
-	return head, parents, prefix
 }
 
 func (s *stack) push(level treeLevel) {