From aea855e8f3823a7ed8d6b5b4daa3fbd475cdf3ee Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 29 Apr 2022 13:06:10 +0300 Subject: [PATCH] [#1326] services/tree: Implement GetSubTree RPC Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/tree.go | 14 +++ pkg/local_object_storage/pilorama/boltdb.go | 25 +++++ pkg/local_object_storage/pilorama/forest.go | 18 ++++ .../pilorama/forest_test.go | 56 ++++++++++ .../pilorama/interface.go | 2 + pkg/local_object_storage/shard/tree.go | 5 + pkg/services/tree/service.go | 99 ++++++++++++++---- pkg/services/tree/service.pb.go | Bin 84811 -> 83819 bytes pkg/services/tree/service.proto | 16 +-- pkg/services/tree/service_grpc.pb.go | Bin 12685 -> 13171 bytes 10 files changed, 209 insertions(+), 26 deletions(-) diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 986bea15e1..04362b8c6c 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -76,3 +76,17 @@ func (e *StorageEngine) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID piloram } return pilorama.Meta{}, err } + +// TreeGetChildren implements the pilorama.Forest interface. +func (e *StorageEngine) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { + var err error + var nodes []uint64 + for _, sh := range e.sortShardsByWeight(cid) { + nodes, err = sh.TreeGetChildren(cid, treeID, nodeID) + if err != nil { + continue + } + return nodes, nil + } + return nil, err +} diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 6eb12a6c6e..808ab18e62 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -416,6 +416,31 @@ func (t *boltForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Met return m, err } +// TreeGetChildren implements the Forest interface. +func (t *boltForest) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) { + key := make([]byte, 9) + key[0] = 'c' + binary.LittleEndian.PutUint64(key[1:], nodeID) + + var children []uint64 + + err := t.db.View(func(tx *bbolt.Tx) error { + treeRoot := tx.Bucket(bucketName(cid, treeID)) + if treeRoot == nil { + return ErrTreeNotFound + } + + b := treeRoot.Bucket(dataBucket) + c := b.Cursor() + for k, _ := c.Seek(key); len(k) == 17 && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { + children = append(children, binary.LittleEndian.Uint64(k[9:])) + } + return nil + }) + + return children, err +} + func (t *boltForest) getPathPrefix(bTree *bbolt.Bucket, attr string, path []string) (int, Node, error) { var key [9]byte diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 53d044866a..f6d79413d7 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -112,3 +112,21 @@ func (f *memoryForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (M return s.getMeta(nodeID), nil } + +// TreeGetChildren implements the Forest interface. +func (f *memoryForest) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) { + fullID := cid.String() + "/" + treeID + s, ok := f.treeMap[fullID] + if !ok { + return nil, ErrTreeNotFound + } + + children, ok := s.childMap[nodeID] + if !ok { + return nil, nil + } + + res := make([]Node, len(children)) + copy(res, children) + return res, nil +} diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index f3a498fe50..d3503bf3b1 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -98,6 +98,62 @@ func testForestTreeMove(t *testing.T, s Forest) { }) } +func TestMemoryForest_TreeGetChildren(t *testing.T) { + for i := range providers { + t.Run(providers[i].name, func(t *testing.T) { + testForestTreeGetChildren(t, providers[i].construct(t)) + }) + } +} + +func testForestTreeGetChildren(t *testing.T, s Forest) { + cid := cidtest.ID() + treeID := "version" + + treeAdd := func(t *testing.T, child, parent Node) { + _, err := s.TreeMove(cid, treeID, &Move{ + Parent: parent, + Child: child, + }) + require.NoError(t, err) + } + + // 0 + // |- 10 + // | |- 3 + // | |- 6 + // | |- 11 + // |- 2 + // |- 7 + treeAdd(t, 10, 0) + treeAdd(t, 3, 10) + treeAdd(t, 6, 10) + treeAdd(t, 11, 6) + treeAdd(t, 2, 0) + treeAdd(t, 7, 0) + + testGetChildren := func(t *testing.T, nodeID Node, expected []Node) { + actual, err := s.TreeGetChildren(cid, treeID, nodeID) + require.NoError(t, err) + require.ElementsMatch(t, expected, actual) + } + + testGetChildren(t, 0, []uint64{10, 2, 7}) + testGetChildren(t, 10, []uint64{3, 6}) + testGetChildren(t, 3, nil) + testGetChildren(t, 6, []uint64{11}) + testGetChildren(t, 11, nil) + testGetChildren(t, 2, nil) + testGetChildren(t, 7, nil) + t.Run("missing node", func(t *testing.T) { + testGetChildren(t, 42, nil) + }) + t.Run("missing tree", func(t *testing.T) { + _, err := s.TreeGetChildren(cid, treeID+"123", 0) + require.ErrorIs(t, err, ErrTreeNotFound) + }) +} + func TestForest_TreeAdd(t *testing.T) { for i := range providers { t.Run(providers[i].name, func(t *testing.T) { diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 005ce5ce5a..9ffa529a81 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -20,6 +20,8 @@ type Forest interface { TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) // TreeGetMeta returns meta information of the node with the specified ID. TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, error) + // TreeGetChildren returns children of the node with the specified ID. The order is arbitrary. + TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) } type ForestStorage interface { diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 2a4120a12d..13cd8fcf00 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -31,3 +31,8 @@ func (s *Shard) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path [] func (s *Shard) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, error) { return s.pilorama.TreeGetMeta(cid, treeID, nodeID) } + +// TreeGetChildren implements the pilorama.Forest interface. +func (s *Shard) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { + return s.pilorama.TreeGetChildren(cid, treeID, nodeID) +} diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 00de22b8ad..8cf6236cd6 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -21,6 +21,9 @@ type Service struct { closeCh chan struct{} } +// MaxGetSubTreeDepth represents maximum allowed traversal depth in GetSubTree RPC. +const MaxGetSubTreeDepth = 10 + var _ TreeServiceServer = (*Service)(nil) // New creates new tree service instance. @@ -66,7 +69,7 @@ func (s *Service) Add(_ context.Context, req *AddRequest) (*AddResponse, error) log, err := s.forest.TreeMove(cid, b.GetTreeId(), &pilorama.Move{ Parent: b.GetParentId(), Child: pilorama.RootID, - Meta: pilorama.Meta{Items: constructMeta(b.GetMeta())}, + Meta: pilorama.Meta{Items: protoToMeta(b.GetMeta())}, }) if err != nil { return nil, err @@ -93,7 +96,7 @@ func (s *Service) AddByPath(_ context.Context, req *AddByPathRequest) (*AddByPat return nil, err } - meta := constructMeta(b.GetMeta()) + meta := protoToMeta(b.GetMeta()) attr := b.GetPathAttribute() if len(attr) == 0 { @@ -173,7 +176,7 @@ func (s *Service) Move(_ context.Context, req *MoveRequest) (*MoveResponse, erro log, err := s.forest.TreeMove(cid, b.GetTreeId(), &pilorama.Move{ Parent: b.GetParentId(), Child: b.GetNodeId(), - Meta: pilorama.Meta{Items: constructMeta(b.GetMeta())}, + Meta: pilorama.Meta{Items: protoToMeta(b.GetMeta())}, }) if err != nil { return nil, err @@ -211,23 +214,20 @@ func (s *Service) GetNodeByPath(_ context.Context, req *GetNodeByPathRequest) (* var x GetNodeByPathResponse_Info x.NodeId = node x.Timestamp = m.Time - for _, kv := range m.Items { - needAttr := b.AllAttributes - if !needAttr { + if b.AllAttributes { + x.Meta = metaToProto(m.Items) + } else { + for _, kv := range m.Items { for _, attr := range b.GetAttributes() { if kv.Key == attr { - needAttr = true + x.Meta = append(x.Meta, &KeyValue{ + Key: kv.Key, + Value: kv.Value, + }) break } } } - - if needAttr { - x.Meta = append(x.Meta, &KeyValue{ - Key: kv.Key, - Value: kv.Value, - }) - } } info = append(info, &x) } @@ -239,8 +239,56 @@ func (s *Service) GetNodeByPath(_ context.Context, req *GetNodeByPathRequest) (* }, nil } -func (s *Service) GetSubTree(_ context.Context, req *GetSubTreeRequest) (*GetSubTreeResponse, error) { - return nil, errors.New("GetSubTree is unimplemented") +type nodeDepthPair struct { + nodes []uint64 + depth uint32 +} + +func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeServer) error { + b := req.GetBody() + if b.GetDepth() > MaxGetSubTreeDepth { + return fmt.Errorf("too big depth: max=%d, got=%d", MaxGetSubTreeDepth, b.GetDepth()) + } + + var cid cidSDK.ID + if err := cid.Decode(b.GetContainerId()); err != nil { + return err + } + + queue := []nodeDepthPair{{[]uint64{b.GetRootId()}, 0}} + + for len(queue) != 0 { + for _, nodeID := range queue[0].nodes { + m, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), nodeID) + if err != nil { + return err + } + err = srv.Send(&GetSubTreeResponse{ + Body: &GetSubTreeResponse_Body{ + NodeId: b.GetRootId(), + ParentId: b.GetRootId(), + Timestamp: m.Time, + Meta: metaToProto(m.Items), + }, + }) + if err != nil { + return err + } + } + + if queue[0].depth < b.GetDepth() { + for _, nodeID := range queue[0].nodes { + children, err := s.forest.TreeGetChildren(cid, b.GetTreeId(), nodeID) + if err != nil { + return err + } + queue = append(queue, nodeDepthPair{children, queue[0].depth + 1}) + } + } + + queue = queue[1:] + } + return nil } // Apply locally applies operation from the remote node to the tree. @@ -284,11 +332,24 @@ loop: }) } -func constructMeta(arr []*KeyValue) []pilorama.KeyValue { +func protoToMeta(arr []*KeyValue) []pilorama.KeyValue { meta := make([]pilorama.KeyValue, len(arr)) for i, kv := range arr { - meta[i].Key = kv.Key - meta[i].Value = kv.Value + if kv != nil { + meta[i].Key = kv.Key + meta[i].Value = kv.Value + } + } + return meta +} + +func metaToProto(arr []pilorama.KeyValue) []*KeyValue { + meta := make([]*KeyValue, len(arr)) + for i, kv := range arr { + meta[i] = &KeyValue{ + Key: kv.Key, + Value: kv.Value, + } } return meta } diff --git a/pkg/services/tree/service.pb.go b/pkg/services/tree/service.pb.go index bf1f26904e17e6e6bbdfdcbcb98729981b5d02ce..5f0afdb9a21591e6b7d1931ef8aa65caed140a88 100644 GIT binary patch delta 2502 zcmZuydr*|u6`!;0n`J@R<-NNfyC5#buhMnWp9;v}On7Hq67RjZgu4S@iO&0t#sz4u#4wUhnhyXXGS zIlptxJ?Gv%|FUYVRn@K8kEtzZl|QJjZ^VhMnI#Qecywb;=B6j>>%J#8B{i9WhWf^O zk6jX2o_)che(T1@+O2`cW_y)hS5zBld_oHSp4wQ~=y9XGEe2K1;R!WD^?w{PhA%1- zdR81bq6|f!z!J4cqofAA_gNr4phH!G3UeKLqs+d!Gq}OfN>l$hspK7R+ zT>9;LePeT=R>DYU45XtiI9Q>k(`a?bLZ?QrLSul^}x5nV@-b;K^qv#;pT)sT-v z`wTqoNY-6UYRqkZ=w6_FU&a5l1MA9q@_zHU0>6H}7InXm!MoK(_)bYUjK?$>+?kAn zzl-6k3VWVihedCGg8bu}o{zf|!*KAeEtq;c0*1;W^p_e?|F#B0Ooge&IPRq3x=olq zHjexnt}H^fbG=aAGkM&Wqp0=wM14M}Q24WZMkWss-W-aKaA!)crko9- zFIqly;MIOps$S=6a(M;llMP%sLiXQlTsLgVlx6B!CF9{$Ht?EX@VNA6_cF9Qx}rV}BJMKPn6 zOkB@w8d{9C6A^4Oj7;brkL{5@dXyCYTI{&wMCVXEI!>GM;;2iI{bAGr*H=13{Vf`+ z`%?t;^ zs=q|goc_p+2hXlRYtSIJ`_3-It?6_&rKRQQ7*Hc&#(>H@Cb&+8W7m~3NN3c-?&2LE z+6E%=%tVsd9zIinfvGWeLtV4Bnfe5 zW-+Q}BJuN!QD~V-Wc@@I9OuGzC7Ch#sT$)mks(8$MhpW#8upTwBo!YZ?w>vDK+D(B zDCaF}`MVSEUoc{QFd0#^Mp3@yn^vLacB0^^{-Y0jKFC1w=%V=w<<`4y_~z>~VE%@IR`7fAZ#7h~uVq(TH(y-M^3f+NEh4A!%Q*fwRzml|A7qlbtN-4`xk(eKL zQAKg6yOn|NkM-=3iicX#ABDL-4N7lShSKNf7IZ58B1HMERS59ew4RH`^V7+48b+@? z-t+gq8bjA&nN3TPSbEza$XKz4(h&&i#g#0PDd#(`UWg?$61QJt_qgPa$&A_Z3p0;= z^QwqJvWU!&X{iv#J62{@kb{}S$ce~V(NnffM}-nUP@nAN3Ee)wd`WY~m%k_6>QD1l zIdTg^U$=aTa~7U0ZspPQDEvuKxHn+iJ29Ld6TNPn?~R1^jG2w;<+14fN>A$*VeVY# z%Mp~(+Y&+lQj7lu_I)#jvz;cgu_+VX$0xyNru&)I%%>n=Ca=Ps%Z`{Sk6diTOu6ia znY7FvOS^>Px3TnsSlw~dD%OEGY8Puod}u!!PrJpwDIsJZmPk9res?0hBGx5I)G5{r zNpwK0o0I85Eq}YSvLv^g?TDfTR+2#RY&@9~bZ&k!Lg5K1jwX_mXIpNeV&VD=3q8Qf zEPS?ZTBuMUX;v!YS#PpZ0bjbTv{J)c#~D2AYBVQ}vXMhOk7To~B#LE2HhvaAw$bz4 z;ku1d#E~e4VFr1{xx}*$iJy^=owidP`$VF2$`NubbEJ@$XOnHG7`8cuve_dkWMbVZ zl&SFK3OJTcIym_p*Ku+Vtl>u7)x`GNIi*|31?-gIjdyTXa&XmS4)XA5GArj&v9}zQ z!PVWMD%p=>zu6N-FOiTSdEw MH=+0G68ba!7bjaqV*mgE delta 2246 zcmZuydvH|c6`!-&z58r7+3e=AkDI$IBy3#B-Q8sKuoO1o5yVkYbP}UHED08swFbtZ zrOI@s#ZC!{$9P5_2B%d<0SU7>5lu``VH5<>w#r~MfzisNZ4z6kW+U``cVp?O|J?6> z=bZ2NJCEOY@5MI5v8N5)#%<_*={?iUiIq()ilQK_DugKdS5@GZ1{?ZnZMbuXE!n-j zO^;jen~VMV27F_-0~fb#!kYCm_BF)u+1pI`_8)x^!Nl@vKH@-EyBinP*fHL|6kqvM zE4n*w!u8K*;rfoLsMOlAwZn$J9rfsc;cK*`iv3+?e7sh|#dt3c>?*=DSg*zPK2=YL z;^4&VJAdU^u%+V_dP*W53b(ZA)g3X}^A4;rEL<7p|mc*LsfH2`$ zLxISJPR2pP?Je0ju-!&S<6yckyfQiQ={CSSMsl#_T^UQyOE~3RK8E^DI*S=szgwh_L_!Ll9w`_5 ztkEDYKc+`@?7HkQ+tt=+SUlAgA@+{-2eJ440QQYK=?@t&4IjN=C$j`Mq5X^LxbK() zhrhBUPxSaeR~E&G|F!_TPB<_$l!u!?6O3Ovn{{}ZFuz?Cn)&wcoE((ZpW#w`mlLC6HlbP z*fH+LJ5S}}z_?3`(Axi#R1mEb)9D`oMm%uzF+88Le~-sHF+~hYod_no2j>8uIPJoT z_i}Ti+O(=FMM-K@d#>QQ4t=WuyRHN~Yrus|uJ}cxu&&304-DqhGJq_6=Sn8ZUwJiS zXsp=4gJ*`yt3jN3#E+$iC4AzleKNv<)knhgFA1jNQo_qj8R~nzpNIr4{geH~d6cIxr;-e2;BIpxzSG6rl3m=ovJ@Qe8$=4dTWFB|WRG@QtK>VrZoU zN+j{IVVvGLxAFcgut=q)N~EHicKRWoHanmcDyh!_HBdzxJ>Vi&4os&FIbf!RIS_>~ zJ(dGi5TS!P(5{QBw9pB^5$j1Ow25`O3p!0vHKZ_Z1+B0#YFhzJqnWu-C;Y$5h0W$k ze>hBMY|L@a$pLrf!CVpWTpnx^>-G8Y%>M!_XgC{kX<0U3(qFrvn6e6>K}6kG0NcfC zc0(Qg(+x&lWU`hobD4Ui0BqFHW@>YQWSQiRL?>eeu`W_cBknYJ%mcMt_#+;e{~dLt zBHf1}4@~3gO@BnvwTWq=B}VRh4CMhc^+qFaMFI6d9Ewu zw-E&ll#;nE>k7H(>O%M_y(JvKFXU<@gLjaurXO6~%S)peFVEci&oKp~3J>Lf4FLGM7A5_6%+FAoQQojn?Z~G{{ gRtt;iVhy}c;Th~04ui4BItQKrmO{_dbK$S>UlYjcpa1{> diff --git a/pkg/services/tree/service.proto b/pkg/services/tree/service.proto index f18b835f7e..0da34f652d 100644 --- a/pkg/services/tree/service.proto +++ b/pkg/services/tree/service.proto @@ -21,7 +21,7 @@ service TreeService { // GetNodeByPath returns list of IDs corresponding to a specific filepath. rpc GetNodeByPath (GetNodeByPathRequest) returns (GetNodeByPathResponse); // GetSubTree returns tree corresponding to a specific node. - rpc GetSubTree (GetSubTreeRequest) returns (GetSubTreeResponse); + rpc GetSubTree (GetSubTreeRequest) returns (stream GetSubTreeResponse); /* Synchronization API */ @@ -153,7 +153,10 @@ message GetSubTreeRequest { message Body { bytes container_id = 1; string tree_id = 2; - repeated uint64 nodes = 3; + uint64 root_id = 3; + // Optional depth of the traversal. Zero means return only root. + // Maximum depth is 10. + uint32 depth = 4; } Body body = 1; @@ -161,12 +164,11 @@ message GetSubTreeRequest { } message GetSubTreeResponse { - message Info { - uint64 node_id = 1; - repeated KeyValue meta = 2; - } message Body { - repeated Info info = 1; + uint64 node_id = 1; + uint64 parent_id = 2; + uint64 timestamp = 3; + repeated KeyValue meta = 4; } Body body = 1; diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index 47aac0f78898f9abe608f7dc20df0b610b135f5c..dc6fe4fd1921cf6a75430f5c7522ad32ca4a2ae4 100644 GIT binary patch delta 1157 zcmbW0L2DC16vqi^47;L1Nhy^!K1*13(`8BNp%CdoDtZtrG+wk8iPM*|kljsZX4{k! za`Peuan6F^NqQ0T6}H+Gj*lT1QTe?f`Q@*d#bP*5=mJt2Q3uXVgK>(jAUT1?HV}^xZx>IS z$+1m%&joApu2J*H$cU#!Y`=c+skA1DQGeD`cUZD^3^~hquKV>(!*44JrSEx~Uti}n zF+6NC~?pn5DrxP*RAUw34haxi8s}eHwJ7(WB`B@iXk~akqck+vcJtQZ#cvXhM z5$%$xzG?LwdXV%n;c9Q0_3bz$VE3Rl=MD!HL{3pu#K&6;xj_89y;{9?9ZD_NVF!9v zDgSI(!w;p)g{@jT9PZ(9jEuY4vnOt#G(7Qrcv@~09{p#rnw=t#X*kT?%a^L!Pe|6G zvVm0FIO?V3fjD!lyISZ?kF!*iM5fU<#g__*-?f_q8O7}U=*!3XaZ0xo4b{roP|ViP4~14ckiZ47eLdgR=H-i-Y%+c^ zb|3~1mWFTEUlc@ccT2Z3s454)twA@omf=iP^ic(X9QVUB{i%T>WCL8F8fXv>! zLCk`c1T7%j`=!h`hss>%o9v)g3v`CuWP8mFcBloDH)^K9+45QyvQTl2$k<8>yBi_k;#$uDBjbeorqGQvG3X=7LONvqxb6rx4ldC5? H7#jlsvmRjN