[#1753] services/tree: Do not restrict depth in GetSubTree

Previously, the depth was restricted because with BFS the amount of
nodes we have in memory blows up exponentially. With DFS is is linear,
so we can process trees of arbitrary depth.

Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>
This commit is contained in:
Evgenii Stratonikov 2022-09-05 14:00:23 +03:00 committed by fyrchik
parent c4c2840b52
commit 9840936a4f
2 changed files with 179 additions and 34 deletions

View file

@ -0,0 +1,141 @@
package tree
import (
"errors"
"testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
)
func TestGetSubTree(t *testing.T) {
d := pilorama.CIDDescriptor{CID: cidtest.ID(), Size: 1}
treeID := "sometree"
p := pilorama.NewMemoryForest()
tree := []struct {
path []string
id uint64
}{
{path: []string{"dir1"}},
{path: []string{"dir2"}},
{path: []string{"dir1", "sub1"}},
{path: []string{"dir2", "sub1"}},
{path: []string{"dir2", "sub2"}},
{path: []string{"dir2", "sub1", "subsub1"}},
}
for i := range tree {
path := tree[i].path
meta := []pilorama.KeyValue{
{Key: pilorama.AttributeFilename, Value: []byte(path[len(path)-1])}}
lm, err := p.TreeAddByPath(d, treeID, pilorama.AttributeFilename, path[:len(path)-1], meta)
require.NoError(t, err)
require.Equal(t, 1, len(lm))
tree[i].id = lm[0].Child
}
testGetSubTree := func(t *testing.T, rootID uint64, depth uint32, errIndex int) []uint64 {
acc := subTreeAcc{errIndex: errIndex}
err := getSubTree(&acc, d.CID, &GetSubTreeRequest_Body{
TreeId: treeID,
RootId: rootID,
Depth: depth,
}, p)
if errIndex == -1 {
require.NoError(t, err)
} else {
require.ErrorIs(t, err, errSubTreeSend)
}
// GetSubTree must return child only after is has returned the parent.
require.Equal(t, rootID, acc.seen[0].Body.NodeId)
loop:
for i := 1; i < len(acc.seen); i++ {
parent := acc.seen[i].Body.ParentId
for j := 0; j < i; j++ {
if acc.seen[j].Body.NodeId == parent {
continue loop
}
}
require.Fail(t, "node has parent %d, but it hasn't been seen", parent)
}
// GetSubTree must return valid meta.
for i := range acc.seen {
b := acc.seen[i].Body
meta, node, err := p.TreeGetMeta(d.CID, treeID, b.NodeId)
require.NoError(t, err)
require.Equal(t, node, b.ParentId)
require.Equal(t, meta.Time, b.Timestamp)
require.Equal(t, metaToProto(meta.Items), b.Meta)
}
ordered := make([]uint64, len(acc.seen))
for i := range acc.seen {
ordered[i] = acc.seen[i].Body.NodeId
}
return ordered
}
t.Run("depth = 1, only root", func(t *testing.T) {
actual := testGetSubTree(t, 0, 1, -1)
require.Equal(t, []uint64{0}, actual)
t.Run("custom root", func(t *testing.T) {
actual := testGetSubTree(t, tree[2].id, 1, -1)
require.Equal(t, []uint64{tree[2].id}, actual)
})
})
t.Run("depth = 2", func(t *testing.T) {
actual := testGetSubTree(t, 0, 2, -1)
require.Equal(t, []uint64{0, tree[0].id, tree[1].id}, actual)
t.Run("error in the middle", func(t *testing.T) {
actual := testGetSubTree(t, 0, 2, 0)
require.Equal(t, []uint64{0}, actual)
actual = testGetSubTree(t, 0, 2, 1)
require.Equal(t, []uint64{0, tree[0].id}, actual)
})
})
t.Run("depth = 0 (unrestricted)", func(t *testing.T) {
actual := testGetSubTree(t, 0, 0, -1)
expected := []uint64{
0,
tree[0].id, // dir1
tree[2].id, // dir1/sub1
tree[1].id, // dir2
tree[3].id, // dir2/sub1
tree[5].id, // dir2/sub1/subsub1
tree[4].id, // dir2/sub2
}
require.Equal(t, expected, actual)
})
}
var errSubTreeSend = errors.New("test error")
type subTreeAcc struct {
grpc.ServerStream // to satisfy the interface
// IDs of the seen nodes.
seen []*GetSubTreeResponse
errIndex int
}
func (s *subTreeAcc) Send(r *GetSubTreeResponse) error {
s.seen = append(s.seen, r)
if s.errIndex >= 0 {
if len(s.seen) == s.errIndex+1 {
return errSubTreeSend
}
if s.errIndex >= 0 && len(s.seen) > s.errIndex {
panic("called Send after an error was returned")
}
}
return nil
}

View file

@ -344,16 +344,8 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest)
}, nil }, nil
} }
type nodeDepthPair struct {
nodes []uint64
depth uint32
}
func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeServer) error { func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeServer) error {
b := req.GetBody() b := req.GetBody()
if b.GetDepth() > MaxGetSubTreeDepth {
return fmt.Errorf("too big depth: max=%d, got=%d", MaxGetSubTreeDepth, b.GetDepth())
}
var cid cidSDK.ID var cid cidSDK.ID
if err := cid.Decode(b.GetContainerId()); err != nil { if err := cid.Decode(b.GetContainerId()); err != nil {
@ -389,38 +381,50 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS
return nil return nil
} }
queue := []nodeDepthPair{{[]uint64{b.GetRootId()}, 0}} return getSubTree(srv, cid, b, s.forest)
}
for len(queue) != 0 { func getSubTree(srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error {
for _, nodeID := range queue[0].nodes { // Traverse the tree in a DFS manner. Because we need to support arbitrary depth,
m, p, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), nodeID) // recursive implementation is not suitable here, so we maintain explicit stack.
stack := [][]uint64{{b.GetRootId()}}
for {
if len(stack) == 0 {
break
} else if len(stack[len(stack)-1]) == 0 {
stack = stack[:len(stack)-1]
continue
}
nodeID := stack[len(stack)-1][0]
stack[len(stack)-1] = stack[len(stack)-1][1:]
m, p, err := forest.TreeGetMeta(cid, b.GetTreeId(), nodeID)
if err != nil {
return err
}
err = srv.Send(&GetSubTreeResponse{
Body: &GetSubTreeResponse_Body{
NodeId: nodeID,
ParentId: p,
Timestamp: m.Time,
Meta: metaToProto(m.Items),
},
})
if err != nil {
return err
}
if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() {
children, err := forest.TreeGetChildren(cid, b.GetTreeId(), nodeID)
if err != nil { if err != nil {
return err return err
} }
err = srv.Send(&GetSubTreeResponse{ if len(children) != 0 {
Body: &GetSubTreeResponse_Body{ stack = append(stack, children)
NodeId: nodeID,
ParentId: p,
Timestamp: m.Time,
Meta: metaToProto(m.Items),
},
})
if err != nil {
return err
} }
} }
if queue[0].depth < b.GetDepth() {
for _, nodeID := range queue[0].nodes {
children, err := s.forest.TreeGetChildren(cid, b.GetTreeId(), nodeID)
if err != nil {
return err
}
queue = append(queue, nodeDepthPair{children, queue[0].depth + 1})
}
}
queue = queue[1:]
} }
return nil return nil
} }