From 730f14e4eb16c652dbc1dc66de071565a1e1921f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 20 May 2022 11:41:37 +0300 Subject: [PATCH] [#1406] pilorama: Return parent from `TreeGetMeta` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/tree.go | 9 ++--- pkg/local_object_storage/pilorama/boltdb.go | 13 ++++--- pkg/local_object_storage/pilorama/forest.go | 6 ++-- .../pilorama/forest_test.go | 36 ++++++++++--------- .../pilorama/interface.go | 2 +- pkg/local_object_storage/shard/tree.go | 2 +- pkg/services/tree/service.go | 6 ++-- 7 files changed, 41 insertions(+), 33 deletions(-) diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index ee2083a2c..cc15853f4 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -81,11 +81,12 @@ func (e *StorageEngine) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, } // TreeGetMeta implements the pilorama.Forest interface. -func (e *StorageEngine) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, error) { +func (e *StorageEngine) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, uint64, error) { var err error var m pilorama.Meta + var p uint64 for _, sh := range e.sortShardsByWeight(cid) { - m, err = sh.TreeGetMeta(cid, treeID, nodeID) + m, p, err = sh.TreeGetMeta(cid, treeID, nodeID) if err != nil { e.log.Debug("can't put node in a tree", zap.Stringer("cid", cid), @@ -93,9 +94,9 @@ func (e *StorageEngine) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID piloram zap.String("err", err.Error())) continue } - return m, nil + return m, p, nil } - return pilorama.Meta{}, err + return pilorama.Meta{}, 0, err } // TreeGetChildren implements the pilorama.Forest interface. diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 808ab18e6..2cd389d44 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -399,10 +399,12 @@ func (t *boltForest) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, pa } // TreeGetMeta implements the forest interface. -func (t *boltForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, error) { - key := metaKey(make([]byte, 9), nodeID) +func (t *boltForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) { + key := parentKey(make([]byte, 9), nodeID) var m Meta + var parentID uint64 + err := t.db.View(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(bucketName(cid, treeID)) if treeRoot == nil { @@ -410,10 +412,13 @@ func (t *boltForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Met } b := treeRoot.Bucket(dataBucket) - return m.FromBytes(b.Get(key)) + if data := b.Get(key); len(data) == 8 { + parentID = binary.LittleEndian.Uint64(data) + } + return m.FromBytes(b.Get(metaKey(key, nodeID))) }) - return m, err + return m, parentID, err } // TreeGetChildren implements the Forest interface. diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index f6d79413d..1af95c922 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -103,14 +103,14 @@ func (f *memoryForest) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, } // TreeGetMeta implements the Forest interface. -func (f *memoryForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, error) { +func (f *memoryForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { - return Meta{}, ErrTreeNotFound + return Meta{}, 0, ErrTreeNotFound } - return s.getMeta(nodeID), nil + return s.getMeta(nodeID), s.infoMap[nodeID].Parent, nil } // TreeGetChildren implements the Forest interface. diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index c2b12a585..585ca4252 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -41,10 +41,11 @@ var providers = []struct { }}, } -func testMeta(t *testing.T, f Forest, cid cidSDK.ID, treeID string, nodeID Node, expected Meta) { - actual, err := f.TreeGetMeta(cid, treeID, nodeID) +func testMeta(t *testing.T, f Forest, cid cidSDK.ID, treeID string, nodeID, parentID Node, expected Meta) { + actualMeta, actualParent, err := f.TreeGetMeta(cid, treeID, nodeID) require.NoError(t, err) - require.Equal(t, expected, actual) + require.Equal(t, parentID, actualParent) + require.Equal(t, expected, actualMeta) } func TestForest_TreeMove(t *testing.T) { @@ -175,7 +176,7 @@ func testForestTreeAdd(t *testing.T, s Forest) { }) require.NoError(t, err) - testMeta(t, s, cid, treeID, lm.Child, Meta{Time: lm.Time, Items: meta}) + testMeta(t, s, cid, treeID, lm.Child, lm.Parent, Meta{Time: lm.Time, Items: meta}) nodes, err := s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"file.txt"}, false) require.NoError(t, err) @@ -185,7 +186,7 @@ func testForestTreeAdd(t *testing.T, s Forest) { _, err := s.TreeGetByPath(cid, treeID+"123", AttributeFilename, []string{"file.txt"}, false) require.ErrorIs(t, err, ErrTreeNotFound) - _, err = s.TreeGetMeta(cid, treeID+"123", 0) + _, _, err = s.TreeGetMeta(cid, treeID+"123", 0) require.ErrorIs(t, err, ErrTreeNotFound) }) } @@ -208,11 +209,11 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { lm, err := s.TreeAddByPath(cid, treeID, AttributeFilename, []string{"path", "to"}, meta) require.NoError(t, err) require.Equal(t, 3, len(lm)) - testMeta(t, s, cid, treeID, lm[0].Child, Meta{Time: lm[0].Time, Items: []KeyValue{{AttributeFilename, []byte("path")}}}) - testMeta(t, s, cid, treeID, lm[1].Child, Meta{Time: lm[1].Time, Items: []KeyValue{{AttributeFilename, []byte("to")}}}) + testMeta(t, s, cid, treeID, lm[0].Child, lm[0].Parent, Meta{Time: lm[0].Time, Items: []KeyValue{{AttributeFilename, []byte("path")}}}) + testMeta(t, s, cid, treeID, lm[1].Child, lm[1].Parent, Meta{Time: lm[1].Time, Items: []KeyValue{{AttributeFilename, []byte("to")}}}) firstID := lm[2].Child - testMeta(t, s, cid, treeID, firstID, Meta{Time: lm[2].Time, Items: meta}) + testMeta(t, s, cid, treeID, firstID, lm[2].Parent, Meta{Time: lm[2].Time, Items: meta}) meta[0].Value = []byte("YYY") lm, err = s.TreeAddByPath(cid, treeID, AttributeFilename, []string{"path", "to"}, meta) @@ -220,7 +221,7 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { require.Equal(t, 1, len(lm)) secondID := lm[0].Child - testMeta(t, s, cid, treeID, secondID, Meta{Time: lm[0].Time, Items: meta}) + testMeta(t, s, cid, treeID, secondID, lm[0].Parent, Meta{Time: lm[0].Time, Items: meta}) t.Run("get versions", func(t *testing.T) { // All versions. @@ -239,8 +240,8 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { lm, err = s.TreeAddByPath(cid, treeID, AttributeFilename, []string{"path", "dir"}, meta) require.NoError(t, err) require.Equal(t, 2, len(lm)) - testMeta(t, s, cid, treeID, lm[0].Child, Meta{Time: lm[0].Time, Items: []KeyValue{{AttributeFilename, []byte("dir")}}}) - testMeta(t, s, cid, treeID, lm[1].Child, Meta{Time: lm[1].Time, Items: meta}) + testMeta(t, s, cid, treeID, lm[0].Child, lm[0].Parent, Meta{Time: lm[0].Time, Items: []KeyValue{{AttributeFilename, []byte("dir")}}}) + testMeta(t, s, cid, treeID, lm[1].Child, lm[1].Parent, Meta{Time: lm[1].Time, Items: meta}) } func TestForest_Apply(t *testing.T) { @@ -269,19 +270,19 @@ func testForestTreeApply(t *testing.T, constructor func(t *testing.T) Forest) { meta := Meta{Time: 3, Items: []KeyValue{{"child", []byte{3}}}} testApply(t, s, 11, 10, meta) - testMeta(t, s, cid, treeID, 11, meta) + testMeta(t, s, cid, treeID, 11, 10, meta) testApply(t, s, 10, TrashID, Meta{Time: 2, Items: []KeyValue{{"parent", []byte{2}}}}) - testMeta(t, s, cid, treeID, 11, Meta{}) + testMeta(t, s, cid, treeID, 11, 0, Meta{}) }) t.Run("add a child to non-existent parent, then add a parent", func(t *testing.T) { s := constructor(t) testApply(t, s, 11, 10, Meta{Time: 1, Items: []KeyValue{{"child", []byte{3}}}}) - testMeta(t, s, cid, treeID, 11, Meta{}) + testMeta(t, s, cid, treeID, 11, 0, Meta{}) testApply(t, s, 10, 0, Meta{Time: 2, Items: []KeyValue{{"grand", []byte{1}}}}) - testMeta(t, s, cid, treeID, 11, Meta{}) + testMeta(t, s, cid, treeID, 11, 0, Meta{}) }) } @@ -343,10 +344,11 @@ func testForestTreeApplyRandom(t *testing.T, constructor func(t *testing.T) Fore require.NoError(t, actual.TreeApply(cid, treeID, &ops[i])) } for i := uint64(0); i < nodeCount; i++ { - expectedMeta, err := expected.TreeGetMeta(cid, treeID, i) + expectedMeta, expectedParent, err := expected.TreeGetMeta(cid, treeID, i) require.NoError(t, err) - actualMeta, err := actual.TreeGetMeta(cid, treeID, i) + actualMeta, actualParent, err := actual.TreeGetMeta(cid, treeID, i) require.NoError(t, err) + require.Equal(t, expectedParent, actualParent, "node id: %d", i) require.Equal(t, expectedMeta, actualMeta, "node id: %d", i) if _, ok := actual.(*memoryForest); ok { diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 23d3a2f6f..b0bf5f2bb 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -21,7 +21,7 @@ type Forest interface { TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) // TreeGetMeta returns meta information of the node with the specified ID. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. - TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, error) + TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) // TreeGetChildren returns children of the node with the specified ID. The order is arbitrary. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 13cd8fcf0..d6eefb7be 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -28,7 +28,7 @@ func (s *Shard) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path [] } // TreeGetMeta implements the pilorama.Forest interface. -func (s *Shard) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, error) { +func (s *Shard) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, uint64, error) { return s.pilorama.TreeGetMeta(cid, treeID, nodeID) } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 3947d712a..c9b2e1a97 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -211,7 +211,7 @@ func (s *Service) GetNodeByPath(_ context.Context, req *GetNodeByPathRequest) (* info := make([]*GetNodeByPathResponse_Info, 0, len(nodes)) for _, node := range nodes { - m, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), node) + m, _, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), node) if err != nil { return nil, err } @@ -269,14 +269,14 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS for len(queue) != 0 { for _, nodeID := range queue[0].nodes { - m, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), nodeID) + m, p, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), nodeID) if err != nil { return err } err = srv.Send(&GetSubTreeResponse{ Body: &GetSubTreeResponse_Body{ NodeId: nodeID, - ParentId: b.GetRootId(), + ParentId: p, Timestamp: m.Time, Meta: metaToProto(m.Items), },