TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 93 Pull requests 4 Projects Releases 1 Activity Actions

[#1406] pilorama: Return parent from TreeGetMeta

Browse source 
Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-05-20 11:41:37 +03:00 committed by fyrchik
parent b30f14978d
commit ad48918a97
7 changed files with 41 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
pkg/local_object_storage/engine/tree.go
View file

@ -81,11 +81,12 @@ func (e *StorageEngine) TreeGetByPath(cid cidSDK.ID, treeID string, attr string,
} }
// TreeGetMeta implements the pilorama.Forest interface. // TreeGetMeta implements the pilorama.Forest interface.
func (e *StorageEngine) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, error) { func (e *StorageEngine) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, uint64, error) {
var err error var err error
var m pilorama.Meta var m pilorama.Meta
var p uint64
for _, sh := range e.sortShardsByWeight(cid) { for _, sh := range e.sortShardsByWeight(cid) {
m, err = sh.TreeGetMeta(cid, treeID, nodeID) m, p, err = sh.TreeGetMeta(cid, treeID, nodeID)
if err != nil { if err != nil {
e.log.Debug("can't put node in a tree", e.log.Debug("can't put node in a tree",
zap.Stringer("cid", cid), zap.Stringer("cid", cid),
@ -93,9 +94,9 @@ func (e *StorageEngine) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID piloram
zap.String("err", err.Error())) zap.String("err", err.Error()))
continue continue
} }
return m, nil return m, p, nil
} }
return pilorama.Meta{}, err return pilorama.Meta{}, 0, err
} }
// TreeGetChildren implements the pilorama.Forest interface. // TreeGetChildren implements the pilorama.Forest interface.

13
pkg/local_object_storage/pilorama/boltdb.go
View file

@ -399,10 +399,12 @@ func (t *boltForest) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, pa
} }
// TreeGetMeta implements the forest interface. // TreeGetMeta implements the forest interface.
func (t *boltForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, error) { func (t *boltForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) {
key := metaKey(make([]byte, 9), nodeID) key := parentKey(make([]byte, 9), nodeID)
var m Meta var m Meta
var parentID uint64
err := t.db.View(func(tx *bbolt.Tx) error { err := t.db.View(func(tx *bbolt.Tx) error {
treeRoot := tx.Bucket(bucketName(cid, treeID)) treeRoot := tx.Bucket(bucketName(cid, treeID))
if treeRoot == nil { if treeRoot == nil {
@ -410,10 +412,13 @@ func (t *boltForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Met
} }
b := treeRoot.Bucket(dataBucket) b := treeRoot.Bucket(dataBucket)
return m.FromBytes(b.Get(key)) if data := b.Get(key); len(data) == 8 {
parentID = binary.LittleEndian.Uint64(data)
}
return m.FromBytes(b.Get(metaKey(key, nodeID)))
}) })
return m, err return m, parentID, err
} }
// TreeGetChildren implements the Forest interface. // TreeGetChildren implements the Forest interface.

6
pkg/local_object_storage/pilorama/forest.go
View file

@ -103,14 +103,14 @@ func (f *memoryForest) TreeGetByPath(cid cidSDK.ID, treeID string, attr string,
} }
// TreeGetMeta implements the Forest interface. // TreeGetMeta implements the Forest interface.
func (f *memoryForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, error) { func (f *memoryForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) {
fullID := cid.String() + "/" + treeID fullID := cid.String() + "/" + treeID
s, ok := f.treeMap[fullID] s, ok := f.treeMap[fullID]
if !ok { if !ok {
return Meta{}, ErrTreeNotFound return Meta{}, 0, ErrTreeNotFound
} }
return s.getMeta(nodeID), nil return s.getMeta(nodeID), s.infoMap[nodeID].Parent, nil
} }
// TreeGetChildren implements the Forest interface. // TreeGetChildren implements the Forest interface.

36
pkg/local_object_storage/pilorama/forest_test.go
View file

@ -41,10 +41,11 @@ var providers = []struct {
}}, }},
} }
func testMeta(t *testing.T, f Forest, cid cidSDK.ID, treeID string, nodeID Node, expected Meta) { func testMeta(t *testing.T, f Forest, cid cidSDK.ID, treeID string, nodeID, parentID Node, expected Meta) {
actual, err := f.TreeGetMeta(cid, treeID, nodeID) actualMeta, actualParent, err := f.TreeGetMeta(cid, treeID, nodeID)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, expected, actual) require.Equal(t, parentID, actualParent)
require.Equal(t, expected, actualMeta)
} }
func TestForest_TreeMove(t *testing.T) { func TestForest_TreeMove(t *testing.T) {
@ -175,7 +176,7 @@ func testForestTreeAdd(t *testing.T, s Forest) {
}) })
require.NoError(t, err) require.NoError(t, err)
testMeta(t, s, cid, treeID, lm.Child, Meta{Time: lm.Time, Items: meta}) testMeta(t, s, cid, treeID, lm.Child, lm.Parent, Meta{Time: lm.Time, Items: meta})
nodes, err := s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"file.txt"}, false) nodes, err := s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"file.txt"}, false)
require.NoError(t, err) require.NoError(t, err)
@ -185,7 +186,7 @@ func testForestTreeAdd(t *testing.T, s Forest) {
_, err := s.TreeGetByPath(cid, treeID+"123", AttributeFilename, []string{"file.txt"}, false) _, err := s.TreeGetByPath(cid, treeID+"123", AttributeFilename, []string{"file.txt"}, false)
require.ErrorIs(t, err, ErrTreeNotFound) require.ErrorIs(t, err, ErrTreeNotFound)
_, err = s.TreeGetMeta(cid, treeID+"123", 0) _, _, err = s.TreeGetMeta(cid, treeID+"123", 0)
require.ErrorIs(t, err, ErrTreeNotFound) require.ErrorIs(t, err, ErrTreeNotFound)
}) })
} }
@ -208,11 +209,11 @@ func testForestTreeAddByPath(t *testing.T, s Forest) {
lm, err := s.TreeAddByPath(cid, treeID, AttributeFilename, []string{"path", "to"}, meta) lm, err := s.TreeAddByPath(cid, treeID, AttributeFilename, []string{"path", "to"}, meta)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 3, len(lm)) require.Equal(t, 3, len(lm))
testMeta(t, s, cid, treeID, lm[0].Child, Meta{Time: lm[0].Time, Items: []KeyValue{{AttributeFilename, []byte("path")}}}) testMeta(t, s, cid, treeID, lm[0].Child, lm[0].Parent, Meta{Time: lm[0].Time, Items: []KeyValue{{AttributeFilename, []byte("path")}}})
testMeta(t, s, cid, treeID, lm[1].Child, Meta{Time: lm[1].Time, Items: []KeyValue{{AttributeFilename, []byte("to")}}}) testMeta(t, s, cid, treeID, lm[1].Child, lm[1].Parent, Meta{Time: lm[1].Time, Items: []KeyValue{{AttributeFilename, []byte("to")}}})
firstID := lm[2].Child firstID := lm[2].Child
testMeta(t, s, cid, treeID, firstID, Meta{Time: lm[2].Time, Items: meta}) testMeta(t, s, cid, treeID, firstID, lm[2].Parent, Meta{Time: lm[2].Time, Items: meta})
meta[0].Value = []byte("YYY") meta[0].Value = []byte("YYY")
lm, err = s.TreeAddByPath(cid, treeID, AttributeFilename, []string{"path", "to"}, meta) lm, err = s.TreeAddByPath(cid, treeID, AttributeFilename, []string{"path", "to"}, meta)
@ -220,7 +221,7 @@ func testForestTreeAddByPath(t *testing.T, s Forest) {
require.Equal(t, 1, len(lm)) require.Equal(t, 1, len(lm))
secondID := lm[0].Child secondID := lm[0].Child
testMeta(t, s, cid, treeID, secondID, Meta{Time: lm[0].Time, Items: meta}) testMeta(t, s, cid, treeID, secondID, lm[0].Parent, Meta{Time: lm[0].Time, Items: meta})
t.Run("get versions", func(t *testing.T) { t.Run("get versions", func(t *testing.T) {
// All versions. // All versions.
@ -239,8 +240,8 @@ func testForestTreeAddByPath(t *testing.T, s Forest) {
lm, err = s.TreeAddByPath(cid, treeID, AttributeFilename, []string{"path", "dir"}, meta) lm, err = s.TreeAddByPath(cid, treeID, AttributeFilename, []string{"path", "dir"}, meta)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 2, len(lm)) require.Equal(t, 2, len(lm))
testMeta(t, s, cid, treeID, lm[0].Child, Meta{Time: lm[0].Time, Items: []KeyValue{{AttributeFilename, []byte("dir")}}}) testMeta(t, s, cid, treeID, lm[0].Child, lm[0].Parent, Meta{Time: lm[0].Time, Items: []KeyValue{{AttributeFilename, []byte("dir")}}})
testMeta(t, s, cid, treeID, lm[1].Child, Meta{Time: lm[1].Time, Items: meta}) testMeta(t, s, cid, treeID, lm[1].Child, lm[1].Parent, Meta{Time: lm[1].Time, Items: meta})
} }
func TestForest_Apply(t *testing.T) { func TestForest_Apply(t *testing.T) {
@ -269,19 +270,19 @@ func testForestTreeApply(t *testing.T, constructor func(t *testing.T) Forest) {
meta := Meta{Time: 3, Items: []KeyValue{{"child", []byte{3}}}} meta := Meta{Time: 3, Items: []KeyValue{{"child", []byte{3}}}}
testApply(t, s, 11, 10, meta) testApply(t, s, 11, 10, meta)
testMeta(t, s, cid, treeID, 11, meta) testMeta(t, s, cid, treeID, 11, 10, meta)
testApply(t, s, 10, TrashID, Meta{Time: 2, Items: []KeyValue{{"parent", []byte{2}}}}) testApply(t, s, 10, TrashID, Meta{Time: 2, Items: []KeyValue{{"parent", []byte{2}}}})
testMeta(t, s, cid, treeID, 11, Meta{}) testMeta(t, s, cid, treeID, 11, 0, Meta{})
}) })
t.Run("add a child to non-existent parent, then add a parent", func(t *testing.T) { t.Run("add a child to non-existent parent, then add a parent", func(t *testing.T) {
s := constructor(t) s := constructor(t)
testApply(t, s, 11, 10, Meta{Time: 1, Items: []KeyValue{{"child", []byte{3}}}}) testApply(t, s, 11, 10, Meta{Time: 1, Items: []KeyValue{{"child", []byte{3}}}})
testMeta(t, s, cid, treeID, 11, Meta{}) testMeta(t, s, cid, treeID, 11, 0, Meta{})
testApply(t, s, 10, 0, Meta{Time: 2, Items: []KeyValue{{"grand", []byte{1}}}}) testApply(t, s, 10, 0, Meta{Time: 2, Items: []KeyValue{{"grand", []byte{1}}}})
testMeta(t, s, cid, treeID, 11, Meta{}) testMeta(t, s, cid, treeID, 11, 0, Meta{})
}) })
} }
@ -343,10 +344,11 @@ func testForestTreeApplyRandom(t *testing.T, constructor func(t *testing.T) Fore
require.NoError(t, actual.TreeApply(cid, treeID, &ops[i])) require.NoError(t, actual.TreeApply(cid, treeID, &ops[i]))
} }
for i := uint64(0); i < nodeCount; i++ { for i := uint64(0); i < nodeCount; i++ {
expectedMeta, err := expected.TreeGetMeta(cid, treeID, i) expectedMeta, expectedParent, err := expected.TreeGetMeta(cid, treeID, i)
require.NoError(t, err) require.NoError(t, err)
actualMeta, err := actual.TreeGetMeta(cid, treeID, i) actualMeta, actualParent, err := actual.TreeGetMeta(cid, treeID, i)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, expectedParent, actualParent, "node id: %d", i)
require.Equal(t, expectedMeta, actualMeta, "node id: %d", i) require.Equal(t, expectedMeta, actualMeta, "node id: %d", i)
if _, ok := actual.(*memoryForest); ok { if _, ok := actual.(*memoryForest); ok {

2
pkg/local_object_storage/pilorama/interface.go
View file

@ -21,7 +21,7 @@ type Forest interface {
TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error)
// TreeGetMeta returns meta information of the node with the specified ID. // TreeGetMeta returns meta information of the node with the specified ID.
// Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree.
TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, error) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error)
// TreeGetChildren returns children of the node with the specified ID. The order is arbitrary. // TreeGetChildren returns children of the node with the specified ID. The order is arbitrary.
// Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree.
TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error)

2
pkg/local_object_storage/shard/tree.go
View file

@ -28,7 +28,7 @@ func (s *Shard) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []
} }
// TreeGetMeta implements the pilorama.Forest interface. // TreeGetMeta implements the pilorama.Forest interface.
func (s *Shard) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, error) { func (s *Shard) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, uint64, error) {
return s.pilorama.TreeGetMeta(cid, treeID, nodeID) return s.pilorama.TreeGetMeta(cid, treeID, nodeID)
} }

6
pkg/services/tree/service.go
View file

@ -211,7 +211,7 @@ func (s *Service) GetNodeByPath(_ context.Context, req *GetNodeByPathRequest) (*
info := make([]*GetNodeByPathResponse_Info, 0, len(nodes)) info := make([]*GetNodeByPathResponse_Info, 0, len(nodes))
for _, node := range nodes { for _, node := range nodes {
m, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), node) m, _, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), node)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -269,14 +269,14 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS
for len(queue) != 0 { for len(queue) != 0 {
for _, nodeID := range queue[0].nodes { for _, nodeID := range queue[0].nodes {
m, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), nodeID) m, p, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), nodeID)
if err != nil { if err != nil {
return err return err
} }
err = srv.Send(&GetSubTreeResponse{ err = srv.Send(&GetSubTreeResponse{
Body: &GetSubTreeResponse_Body{ Body: &GetSubTreeResponse_Body{
NodeId: nodeID, NodeId: nodeID,
ParentId: b.GetRootId(), ParentId: p,
Timestamp: m.Time, Timestamp: m.Time,
Meta: metaToProto(m.Items), Meta: metaToProto(m.Items),
}, },