forked from TrueCloudLab/frostfs-node
WIP: Morph: Add unit tests #2
12 changed files with 81 additions and 22 deletions
|
@ -172,7 +172,7 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s
|
||||||
}
|
}
|
||||||
|
|
||||||
// TreeGetChildren implements the pilorama.Forest interface.
|
// TreeGetChildren implements the pilorama.Forest interface.
|
||||||
func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) {
|
func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetChildren",
|
ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetChildren",
|
||||||
trace.WithAttributes(
|
trace.WithAttributes(
|
||||||
attribute.String("container_id", cid.EncodeToString()),
|
attribute.String("container_id", cid.EncodeToString()),
|
||||||
|
@ -183,7 +183,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree
|
||||||
defer span.End()
|
defer span.End()
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
var nodes []uint64
|
var nodes []pilorama.NodeInfo
|
||||||
for _, sh := range e.sortShardsByWeight(cid) {
|
for _, sh := range e.sortShardsByWeight(cid) {
|
||||||
nodes, err = sh.TreeGetChildren(ctx, cid, treeID, nodeID)
|
nodes, err = sh.TreeGetChildren(ctx, cid, treeID, nodeID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -927,7 +927,7 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri
|
||||||
}
|
}
|
||||||
|
|
||||||
// TreeGetChildren implements the Forest interface.
|
// TreeGetChildren implements the Forest interface.
|
||||||
func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) {
|
func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) {
|
||||||
var (
|
var (
|
||||||
startedAt = time.Now()
|
startedAt = time.Now()
|
||||||
success = false
|
success = false
|
||||||
|
@ -956,7 +956,7 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID
|
||||||
key[0] = 'c'
|
key[0] = 'c'
|
||||||
binary.LittleEndian.PutUint64(key[1:], nodeID)
|
binary.LittleEndian.PutUint64(key[1:], nodeID)
|
||||||
|
|
||||||
var children []uint64
|
var result []NodeInfo
|
||||||
|
|
||||||
err := t.db.View(func(tx *bbolt.Tx) error {
|
err := t.db.View(func(tx *bbolt.Tx) error {
|
||||||
treeRoot := tx.Bucket(bucketName(cid, treeID))
|
treeRoot := tx.Bucket(bucketName(cid, treeID))
|
||||||
|
@ -967,12 +967,23 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID
|
||||||
b := treeRoot.Bucket(dataBucket)
|
b := treeRoot.Bucket(dataBucket)
|
||||||
c := b.Cursor()
|
c := b.Cursor()
|
||||||
for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() {
|
for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() {
|
||||||
children = append(children, binary.LittleEndian.Uint64(k[9:]))
|
childID := binary.LittleEndian.Uint64(k[9:])
|
||||||
|
childInfo := NodeInfo{
|
||||||
|
ID: childID,
|
||||||
|
}
|
||||||
|
parentID, _, metaBytes, found := t.getState(b, stateKey(key, childID))
|
||||||
|
if found {
|
||||||
|
childInfo.ParentID = parentID
|
||||||
|
if err := childInfo.Meta.FromBytes(metaBytes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result = append(result, childInfo)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
success = err == nil
|
success = err == nil
|
||||||
return children, metaerr.Wrap(err)
|
return result, metaerr.Wrap(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TreeList implements the Forest interface.
|
// TreeList implements the Forest interface.
|
||||||
|
|
|
@ -148,7 +148,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string,
|
||||||
}
|
}
|
||||||
|
|
||||||
// TreeGetChildren implements the Forest interface.
|
// TreeGetChildren implements the Forest interface.
|
||||||
func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]uint64, error) {
|
func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]NodeInfo, error) {
|
||||||
fullID := cid.String() + "/" + treeID
|
fullID := cid.String() + "/" + treeID
|
||||||
s, ok := f.treeMap[fullID]
|
s, ok := f.treeMap[fullID]
|
||||||
if !ok {
|
if !ok {
|
||||||
|
@ -156,8 +156,14 @@ func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID str
|
||||||
}
|
}
|
||||||
|
|
||||||
children := s.tree.getChildren(nodeID)
|
children := s.tree.getChildren(nodeID)
|
||||||
res := make([]Node, len(children))
|
res := make([]NodeInfo, 0, len(children))
|
||||||
copy(res, children)
|
for _, childID := range children {
|
||||||
|
res = append(res, NodeInfo{
|
||||||
|
ID: childID,
|
||||||
|
Meta: s.infoMap[childID].Meta,
|
||||||
|
ParentID: s.infoMap[childID].Parent,
|
||||||
|
})
|
||||||
|
}
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -32,7 +32,7 @@ type Forest interface {
|
||||||
TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error)
|
TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error)
|
||||||
// TreeGetChildren returns children of the node with the specified ID. The order is arbitrary.
|
// TreeGetChildren returns children of the node with the specified ID. The order is arbitrary.
|
||||||
// Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree.
|
// Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree.
|
||||||
TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error)
|
TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error)
|
||||||
// TreeGetOpLog returns first log operation stored at or above the height.
|
// TreeGetOpLog returns first log operation stored at or above the height.
|
||||||
// In case no such operation is found, empty Move and nil error should be returned.
|
// In case no such operation is found, empty Move and nil error should be returned.
|
||||||
TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error)
|
TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error)
|
||||||
|
|
|
@ -55,3 +55,9 @@ var (
|
||||||
func isAttributeInternal(key string) bool {
|
func isAttributeInternal(key string) bool {
|
||||||
return key == AttributeFilename
|
return key == AttributeFilename
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type NodeInfo struct {
|
||||||
|
ID Node
|
||||||
|
Meta Meta
|
||||||
|
ParentID Node
|
||||||
|
}
|
||||||
|
|
|
@ -159,7 +159,7 @@ func (s *Shard) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, n
|
||||||
}
|
}
|
||||||
|
|
||||||
// TreeGetChildren implements the pilorama.Forest interface.
|
// TreeGetChildren implements the pilorama.Forest interface.
|
||||||
func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) {
|
func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetChildren",
|
ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetChildren",
|
||||||
trace.WithAttributes(
|
trace.WithAttributes(
|
||||||
attribute.String("shard_id", s.ID().String()),
|
attribute.String("shard_id", s.ID().String()),
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
|
@ -440,7 +441,15 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS
|
||||||
func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error {
|
func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error {
|
||||||
// Traverse the tree in a DFS manner. Because we need to support arbitrary depth,
|
// Traverse the tree in a DFS manner. Because we need to support arbitrary depth,
|
||||||
// recursive implementation is not suitable here, so we maintain explicit stack.
|
// recursive implementation is not suitable here, so we maintain explicit stack.
|
||||||
stack := [][]uint64{{b.GetRootId()}}
|
m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
stack := [][]pilorama.NodeInfo{{{
|
||||||
|
ID: b.GetRootId(),
|
||||||
|
Meta: m,
|
||||||
|
ParentID: p,
|
||||||
|
}}}
|
||||||
|
|
||||||
for {
|
for {
|
||||||
if len(stack) == 0 {
|
if len(stack) == 0 {
|
||||||
|
@ -450,19 +459,15 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
nodeID := stack[len(stack)-1][0]
|
node := stack[len(stack)-1][0]
|
||||||
stack[len(stack)-1] = stack[len(stack)-1][1:]
|
stack[len(stack)-1] = stack[len(stack)-1][1:]
|
||||||
|
|
||||||
m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), nodeID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
err = srv.Send(&GetSubTreeResponse{
|
err = srv.Send(&GetSubTreeResponse{
|
||||||
Body: &GetSubTreeResponse_Body{
|
Body: &GetSubTreeResponse_Body{
|
||||||
NodeId: nodeID,
|
NodeId: node.ID,
|
||||||
ParentId: p,
|
ParentId: node.ParentID,
|
||||||
Timestamp: m.Time,
|
Timestamp: node.Meta.Time,
|
||||||
Meta: metaToProto(m.Items),
|
Meta: metaToProto(node.Meta.Items),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -470,7 +475,11 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
|
||||||
}
|
}
|
||||||
|
|
||||||
if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() {
|
if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() {
|
||||||
children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), nodeID)
|
children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), node.ID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
children, err = sortByFilename(children, b.GetOrderBy().GetDirection())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -482,6 +491,24 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func sortByFilename(nodes []pilorama.NodeInfo, d GetSubTreeRequest_Body_Order_Direction) ([]pilorama.NodeInfo, error) {
|
||||||
|
switch d {
|
||||||
|
case GetSubTreeRequest_Body_Order_None:
|
||||||
|
return nodes, nil
|
||||||
|
case GetSubTreeRequest_Body_Order_Asc:
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
return nodes, nil
|
||||||
|
}
|
||||||
|
less := func(i, j int) bool {
|
||||||
|
return bytes.Compare(nodes[i].Meta.GetAttr(pilorama.AttributeFilename), nodes[j].Meta.GetAttr(pilorama.AttributeFilename)) < 0
|
||||||
|
}
|
||||||
|
sort.Slice(nodes, less)
|
||||||
|
return nodes, nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported order direction: %s", d.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Apply locally applies operation from the remote node to the tree.
|
// Apply locally applies operation from the remote node to the tree.
|
||||||
func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) {
|
func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) {
|
||||||
err := verifyMessage(req)
|
err := verifyMessage(req)
|
||||||
|
|
BIN
pkg/services/tree/service.pb.go
generated
BIN
pkg/services/tree/service.pb.go
generated
Binary file not shown.
|
@ -238,6 +238,13 @@ message GetNodeByPathResponse {
|
||||||
|
|
||||||
message GetSubTreeRequest {
|
message GetSubTreeRequest {
|
||||||
message Body {
|
message Body {
|
||||||
|
message Order {
|
||||||
|
enum Direction {
|
||||||
|
None = 0;
|
||||||
|
Asc = 1;
|
||||||
|
}
|
||||||
|
Direction direction = 1;
|
||||||
|
}
|
||||||
// Container ID in V2 format.
|
// Container ID in V2 format.
|
||||||
bytes container_id = 1;
|
bytes container_id = 1;
|
||||||
// The name of the tree.
|
// The name of the tree.
|
||||||
|
@ -249,6 +256,8 @@ message GetSubTreeRequest {
|
||||||
uint32 depth = 4;
|
uint32 depth = 4;
|
||||||
// Bearer token in V2 format.
|
// Bearer token in V2 format.
|
||||||
bytes bearer_token = 5;
|
bytes bearer_token = 5;
|
||||||
|
// Result ordering.
|
||||||
|
Order order_by = 6;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Request body.
|
// Request body.
|
||||||
|
|
BIN
pkg/services/tree/service_frostfs.pb.go
generated
BIN
pkg/services/tree/service_frostfs.pb.go
generated
Binary file not shown.
BIN
pkg/services/tree/service_grpc.pb.go
generated
BIN
pkg/services/tree/service_grpc.pb.go
generated
Binary file not shown.
BIN
pkg/services/tree/types.pb.go
generated
BIN
pkg/services/tree/types.pb.go
generated
Binary file not shown.
Loading…
Reference in a new issue