[#1326] services/tree: Implement GetSubTree RPC

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-04-29 13:06:10 +03:00 committed by fyrchik
parent 62154da17c
commit aea855e8f3
10 changed files with 209 additions and 26 deletions

View file

@ -76,3 +76,17 @@ func (e *StorageEngine) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID piloram
} }
return pilorama.Meta{}, err return pilorama.Meta{}, err
} }
// TreeGetChildren implements the pilorama.Forest interface.
func (e *StorageEngine) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) {
var err error
var nodes []uint64
for _, sh := range e.sortShardsByWeight(cid) {
nodes, err = sh.TreeGetChildren(cid, treeID, nodeID)
if err != nil {
continue
}
return nodes, nil
}
return nil, err
}

View file

@ -416,6 +416,31 @@ func (t *boltForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Met
return m, err return m, err
} }
// TreeGetChildren implements the Forest interface.
func (t *boltForest) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) {
key := make([]byte, 9)
key[0] = 'c'
binary.LittleEndian.PutUint64(key[1:], nodeID)
var children []uint64
err := t.db.View(func(tx *bbolt.Tx) error {
treeRoot := tx.Bucket(bucketName(cid, treeID))
if treeRoot == nil {
return ErrTreeNotFound
}
b := treeRoot.Bucket(dataBucket)
c := b.Cursor()
for k, _ := c.Seek(key); len(k) == 17 && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() {
children = append(children, binary.LittleEndian.Uint64(k[9:]))
}
return nil
})
return children, err
}
func (t *boltForest) getPathPrefix(bTree *bbolt.Bucket, attr string, path []string) (int, Node, error) { func (t *boltForest) getPathPrefix(bTree *bbolt.Bucket, attr string, path []string) (int, Node, error) {
var key [9]byte var key [9]byte

View file

@ -112,3 +112,21 @@ func (f *memoryForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (M
return s.getMeta(nodeID), nil return s.getMeta(nodeID), nil
} }
// TreeGetChildren implements the Forest interface.
func (f *memoryForest) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) {
fullID := cid.String() + "/" + treeID
s, ok := f.treeMap[fullID]
if !ok {
return nil, ErrTreeNotFound
}
children, ok := s.childMap[nodeID]
if !ok {
return nil, nil
}
res := make([]Node, len(children))
copy(res, children)
return res, nil
}

View file

@ -98,6 +98,62 @@ func testForestTreeMove(t *testing.T, s Forest) {
}) })
} }
func TestMemoryForest_TreeGetChildren(t *testing.T) {
for i := range providers {
t.Run(providers[i].name, func(t *testing.T) {
testForestTreeGetChildren(t, providers[i].construct(t))
})
}
}
func testForestTreeGetChildren(t *testing.T, s Forest) {
cid := cidtest.ID()
treeID := "version"
treeAdd := func(t *testing.T, child, parent Node) {
_, err := s.TreeMove(cid, treeID, &Move{
Parent: parent,
Child: child,
})
require.NoError(t, err)
}
// 0
// |- 10
// | |- 3
// | |- 6
// | |- 11
// |- 2
// |- 7
treeAdd(t, 10, 0)
treeAdd(t, 3, 10)
treeAdd(t, 6, 10)
treeAdd(t, 11, 6)
treeAdd(t, 2, 0)
treeAdd(t, 7, 0)
testGetChildren := func(t *testing.T, nodeID Node, expected []Node) {
actual, err := s.TreeGetChildren(cid, treeID, nodeID)
require.NoError(t, err)
require.ElementsMatch(t, expected, actual)
}
testGetChildren(t, 0, []uint64{10, 2, 7})
testGetChildren(t, 10, []uint64{3, 6})
testGetChildren(t, 3, nil)
testGetChildren(t, 6, []uint64{11})
testGetChildren(t, 11, nil)
testGetChildren(t, 2, nil)
testGetChildren(t, 7, nil)
t.Run("missing node", func(t *testing.T) {
testGetChildren(t, 42, nil)
})
t.Run("missing tree", func(t *testing.T) {
_, err := s.TreeGetChildren(cid, treeID+"123", 0)
require.ErrorIs(t, err, ErrTreeNotFound)
})
}
func TestForest_TreeAdd(t *testing.T) { func TestForest_TreeAdd(t *testing.T) {
for i := range providers { for i := range providers {
t.Run(providers[i].name, func(t *testing.T) { t.Run(providers[i].name, func(t *testing.T) {

View file

@ -20,6 +20,8 @@ type Forest interface {
TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error)
// TreeGetMeta returns meta information of the node with the specified ID. // TreeGetMeta returns meta information of the node with the specified ID.
TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, error) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, error)
// TreeGetChildren returns children of the node with the specified ID. The order is arbitrary.
TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error)
} }
type ForestStorage interface { type ForestStorage interface {

View file

@ -31,3 +31,8 @@ func (s *Shard) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []
func (s *Shard) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, error) { func (s *Shard) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, error) {
return s.pilorama.TreeGetMeta(cid, treeID, nodeID) return s.pilorama.TreeGetMeta(cid, treeID, nodeID)
} }
// TreeGetChildren implements the pilorama.Forest interface.
func (s *Shard) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) {
return s.pilorama.TreeGetChildren(cid, treeID, nodeID)
}

View file

@ -21,6 +21,9 @@ type Service struct {
closeCh chan struct{} closeCh chan struct{}
} }
// MaxGetSubTreeDepth represents maximum allowed traversal depth in GetSubTree RPC.
const MaxGetSubTreeDepth = 10
var _ TreeServiceServer = (*Service)(nil) var _ TreeServiceServer = (*Service)(nil)
// New creates new tree service instance. // New creates new tree service instance.
@ -66,7 +69,7 @@ func (s *Service) Add(_ context.Context, req *AddRequest) (*AddResponse, error)
log, err := s.forest.TreeMove(cid, b.GetTreeId(), &pilorama.Move{ log, err := s.forest.TreeMove(cid, b.GetTreeId(), &pilorama.Move{
Parent: b.GetParentId(), Parent: b.GetParentId(),
Child: pilorama.RootID, Child: pilorama.RootID,
Meta: pilorama.Meta{Items: constructMeta(b.GetMeta())}, Meta: pilorama.Meta{Items: protoToMeta(b.GetMeta())},
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -93,7 +96,7 @@ func (s *Service) AddByPath(_ context.Context, req *AddByPathRequest) (*AddByPat
return nil, err return nil, err
} }
meta := constructMeta(b.GetMeta()) meta := protoToMeta(b.GetMeta())
attr := b.GetPathAttribute() attr := b.GetPathAttribute()
if len(attr) == 0 { if len(attr) == 0 {
@ -173,7 +176,7 @@ func (s *Service) Move(_ context.Context, req *MoveRequest) (*MoveResponse, erro
log, err := s.forest.TreeMove(cid, b.GetTreeId(), &pilorama.Move{ log, err := s.forest.TreeMove(cid, b.GetTreeId(), &pilorama.Move{
Parent: b.GetParentId(), Parent: b.GetParentId(),
Child: b.GetNodeId(), Child: b.GetNodeId(),
Meta: pilorama.Meta{Items: constructMeta(b.GetMeta())}, Meta: pilorama.Meta{Items: protoToMeta(b.GetMeta())},
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -211,23 +214,20 @@ func (s *Service) GetNodeByPath(_ context.Context, req *GetNodeByPathRequest) (*
var x GetNodeByPathResponse_Info var x GetNodeByPathResponse_Info
x.NodeId = node x.NodeId = node
x.Timestamp = m.Time x.Timestamp = m.Time
for _, kv := range m.Items { if b.AllAttributes {
needAttr := b.AllAttributes x.Meta = metaToProto(m.Items)
if !needAttr { } else {
for _, kv := range m.Items {
for _, attr := range b.GetAttributes() { for _, attr := range b.GetAttributes() {
if kv.Key == attr { if kv.Key == attr {
needAttr = true x.Meta = append(x.Meta, &KeyValue{
Key: kv.Key,
Value: kv.Value,
})
break break
} }
} }
} }
if needAttr {
x.Meta = append(x.Meta, &KeyValue{
Key: kv.Key,
Value: kv.Value,
})
}
} }
info = append(info, &x) info = append(info, &x)
} }
@ -239,8 +239,56 @@ func (s *Service) GetNodeByPath(_ context.Context, req *GetNodeByPathRequest) (*
}, nil }, nil
} }
func (s *Service) GetSubTree(_ context.Context, req *GetSubTreeRequest) (*GetSubTreeResponse, error) { type nodeDepthPair struct {
return nil, errors.New("GetSubTree is unimplemented") nodes []uint64
depth uint32
}
func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeServer) error {
b := req.GetBody()
if b.GetDepth() > MaxGetSubTreeDepth {
return fmt.Errorf("too big depth: max=%d, got=%d", MaxGetSubTreeDepth, b.GetDepth())
}
var cid cidSDK.ID
if err := cid.Decode(b.GetContainerId()); err != nil {
return err
}
queue := []nodeDepthPair{{[]uint64{b.GetRootId()}, 0}}
for len(queue) != 0 {
for _, nodeID := range queue[0].nodes {
m, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), nodeID)
if err != nil {
return err
}
err = srv.Send(&GetSubTreeResponse{
Body: &GetSubTreeResponse_Body{
NodeId: b.GetRootId(),
ParentId: b.GetRootId(),
Timestamp: m.Time,
Meta: metaToProto(m.Items),
},
})
if err != nil {
return err
}
}
if queue[0].depth < b.GetDepth() {
for _, nodeID := range queue[0].nodes {
children, err := s.forest.TreeGetChildren(cid, b.GetTreeId(), nodeID)
if err != nil {
return err
}
queue = append(queue, nodeDepthPair{children, queue[0].depth + 1})
}
}
queue = queue[1:]
}
return nil
} }
// Apply locally applies operation from the remote node to the tree. // Apply locally applies operation from the remote node to the tree.
@ -284,11 +332,24 @@ loop:
}) })
} }
func constructMeta(arr []*KeyValue) []pilorama.KeyValue { func protoToMeta(arr []*KeyValue) []pilorama.KeyValue {
meta := make([]pilorama.KeyValue, len(arr)) meta := make([]pilorama.KeyValue, len(arr))
for i, kv := range arr { for i, kv := range arr {
meta[i].Key = kv.Key if kv != nil {
meta[i].Value = kv.Value meta[i].Key = kv.Key
meta[i].Value = kv.Value
}
}
return meta
}
func metaToProto(arr []pilorama.KeyValue) []*KeyValue {
meta := make([]*KeyValue, len(arr))
for i, kv := range arr {
meta[i] = &KeyValue{
Key: kv.Key,
Value: kv.Value,
}
} }
return meta return meta
} }

Binary file not shown.

View file

@ -21,7 +21,7 @@ service TreeService {
// GetNodeByPath returns list of IDs corresponding to a specific filepath. // GetNodeByPath returns list of IDs corresponding to a specific filepath.
rpc GetNodeByPath (GetNodeByPathRequest) returns (GetNodeByPathResponse); rpc GetNodeByPath (GetNodeByPathRequest) returns (GetNodeByPathResponse);
// GetSubTree returns tree corresponding to a specific node. // GetSubTree returns tree corresponding to a specific node.
rpc GetSubTree (GetSubTreeRequest) returns (GetSubTreeResponse); rpc GetSubTree (GetSubTreeRequest) returns (stream GetSubTreeResponse);
/* Synchronization API */ /* Synchronization API */
@ -153,7 +153,10 @@ message GetSubTreeRequest {
message Body { message Body {
bytes container_id = 1; bytes container_id = 1;
string tree_id = 2; string tree_id = 2;
repeated uint64 nodes = 3; uint64 root_id = 3;
// Optional depth of the traversal. Zero means return only root.
// Maximum depth is 10.
uint32 depth = 4;
} }
Body body = 1; Body body = 1;
@ -161,12 +164,11 @@ message GetSubTreeRequest {
} }
message GetSubTreeResponse { message GetSubTreeResponse {
message Info {
uint64 node_id = 1;
repeated KeyValue meta = 2;
}
message Body { message Body {
repeated Info info = 1; uint64 node_id = 1;
uint64 parent_id = 2;
uint64 timestamp = 3;
repeated KeyValue meta = 4;
} }
Body body = 1; Body body = 1;

Binary file not shown.