forked from TrueCloudLab/frostfs-node
WIP: Morph: Add unit tests #2
12 changed files with 81 additions and 22 deletions
|
@ -172,7 +172,7 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s
|
|||
}
|
||||
|
||||
// TreeGetChildren implements the pilorama.Forest interface.
|
||||
func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) {
|
||||
func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) {
|
||||
ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetChildren",
|
||||
trace.WithAttributes(
|
||||
attribute.String("container_id", cid.EncodeToString()),
|
||||
|
@ -183,7 +183,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree
|
|||
defer span.End()
|
||||
|
||||
var err error
|
||||
var nodes []uint64
|
||||
var nodes []pilorama.NodeInfo
|
||||
for _, sh := range e.sortShardsByWeight(cid) {
|
||||
nodes, err = sh.TreeGetChildren(ctx, cid, treeID, nodeID)
|
||||
if err != nil {
|
||||
|
|
|
@ -927,7 +927,7 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri
|
|||
}
|
||||
|
||||
// TreeGetChildren implements the Forest interface.
|
||||
func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) {
|
||||
func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) {
|
||||
var (
|
||||
startedAt = time.Now()
|
||||
success = false
|
||||
|
@ -956,7 +956,7 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID
|
|||
key[0] = 'c'
|
||||
binary.LittleEndian.PutUint64(key[1:], nodeID)
|
||||
|
||||
var children []uint64
|
||||
var result []NodeInfo
|
||||
|
||||
err := t.db.View(func(tx *bbolt.Tx) error {
|
||||
treeRoot := tx.Bucket(bucketName(cid, treeID))
|
||||
|
@ -967,12 +967,23 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID
|
|||
b := treeRoot.Bucket(dataBucket)
|
||||
c := b.Cursor()
|
||||
for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() {
|
||||
children = append(children, binary.LittleEndian.Uint64(k[9:]))
|
||||
childID := binary.LittleEndian.Uint64(k[9:])
|
||||
childInfo := NodeInfo{
|
||||
ID: childID,
|
||||
}
|
||||
parentID, _, metaBytes, found := t.getState(b, stateKey(key, childID))
|
||||
if found {
|
||||
childInfo.ParentID = parentID
|
||||
if err := childInfo.Meta.FromBytes(metaBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
result = append(result, childInfo)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
success = err == nil
|
||||
return children, metaerr.Wrap(err)
|
||||
return result, metaerr.Wrap(err)
|
||||
}
|
||||
|
||||
// TreeList implements the Forest interface.
|
||||
|
|
|
@ -148,7 +148,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string,
|
|||
}
|
||||
|
||||
// TreeGetChildren implements the Forest interface.
|
||||
func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]uint64, error) {
|
||||
func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]NodeInfo, error) {
|
||||
fullID := cid.String() + "/" + treeID
|
||||
s, ok := f.treeMap[fullID]
|
||||
if !ok {
|
||||
|
@ -156,8 +156,14 @@ func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID str
|
|||
}
|
||||
|
||||
children := s.tree.getChildren(nodeID)
|
||||
res := make([]Node, len(children))
|
||||
copy(res, children)
|
||||
res := make([]NodeInfo, 0, len(children))
|
||||
for _, childID := range children {
|
||||
res = append(res, NodeInfo{
|
||||
ID: childID,
|
||||
Meta: s.infoMap[childID].Meta,
|
||||
ParentID: s.infoMap[childID].Parent,
|
||||
})
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
|
|
|
@ -32,7 +32,7 @@ type Forest interface {
|
|||
TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error)
|
||||
// TreeGetChildren returns children of the node with the specified ID. The order is arbitrary.
|
||||
// Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree.
|
||||
TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error)
|
||||
TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error)
|
||||
// TreeGetOpLog returns first log operation stored at or above the height.
|
||||
// In case no such operation is found, empty Move and nil error should be returned.
|
||||
TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error)
|
||||
|
|
|
@ -55,3 +55,9 @@ var (
|
|||
func isAttributeInternal(key string) bool {
|
||||
return key == AttributeFilename
|
||||
}
|
||||
|
||||
type NodeInfo struct {
|
||||
ID Node
|
||||
Meta Meta
|
||||
ParentID Node
|
||||
}
|
||||
|
|
|
@ -159,7 +159,7 @@ func (s *Shard) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, n
|
|||
}
|
||||
|
||||
// TreeGetChildren implements the pilorama.Forest interface.
|
||||
func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) {
|
||||
func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) {
|
||||
ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetChildren",
|
||||
trace.WithAttributes(
|
||||
attribute.String("shard_id", s.ID().String()),
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
|
@ -440,7 +441,15 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS
|
|||
func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error {
|
||||
// Traverse the tree in a DFS manner. Because we need to support arbitrary depth,
|
||||
// recursive implementation is not suitable here, so we maintain explicit stack.
|
||||
stack := [][]uint64{{b.GetRootId()}}
|
||||
m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stack := [][]pilorama.NodeInfo{{{
|
||||
ID: b.GetRootId(),
|
||||
Meta: m,
|
||||
ParentID: p,
|
||||
}}}
|
||||
|
||||
for {
|
||||
if len(stack) == 0 {
|
||||
|
@ -450,19 +459,15 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
|
|||
continue
|
||||
}
|
||||
|
||||
nodeID := stack[len(stack)-1][0]
|
||||
node := stack[len(stack)-1][0]
|
||||
stack[len(stack)-1] = stack[len(stack)-1][1:]
|
||||
|
||||
m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), nodeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = srv.Send(&GetSubTreeResponse{
|
||||
Body: &GetSubTreeResponse_Body{
|
||||
NodeId: nodeID,
|
||||
ParentId: p,
|
||||
Timestamp: m.Time,
|
||||
Meta: metaToProto(m.Items),
|
||||
NodeId: node.ID,
|
||||
ParentId: node.ParentID,
|
||||
Timestamp: node.Meta.Time,
|
||||
Meta: metaToProto(node.Meta.Items),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
|
@ -470,7 +475,11 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
|
|||
}
|
||||
|
||||
if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() {
|
||||
children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), nodeID)
|
||||
children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), node.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
children, err = sortByFilename(children, b.GetOrderBy().GetDirection())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -482,6 +491,24 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
|
|||
return nil
|
||||
}
|
||||
|
||||
func sortByFilename(nodes []pilorama.NodeInfo, d GetSubTreeRequest_Body_Order_Direction) ([]pilorama.NodeInfo, error) {
|
||||
switch d {
|
||||
case GetSubTreeRequest_Body_Order_None:
|
||||
return nodes, nil
|
||||
case GetSubTreeRequest_Body_Order_Asc:
|
||||
if len(nodes) == 0 {
|
||||
return nodes, nil
|
||||
}
|
||||
less := func(i, j int) bool {
|
||||
return bytes.Compare(nodes[i].Meta.GetAttr(pilorama.AttributeFilename), nodes[j].Meta.GetAttr(pilorama.AttributeFilename)) < 0
|
||||
}
|
||||
sort.Slice(nodes, less)
|
||||
return nodes, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported order direction: %s", d.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Apply locally applies operation from the remote node to the tree.
|
||||
func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) {
|
||||
err := verifyMessage(req)
|
||||
|
|
BIN
pkg/services/tree/service.pb.go
generated
BIN
pkg/services/tree/service.pb.go
generated
Binary file not shown.
|
@ -238,6 +238,13 @@ message GetNodeByPathResponse {
|
|||
|
||||
message GetSubTreeRequest {
|
||||
message Body {
|
||||
message Order {
|
||||
enum Direction {
|
||||
None = 0;
|
||||
Asc = 1;
|
||||
}
|
||||
Direction direction = 1;
|
||||
}
|
||||
// Container ID in V2 format.
|
||||
bytes container_id = 1;
|
||||
// The name of the tree.
|
||||
|
@ -249,6 +256,8 @@ message GetSubTreeRequest {
|
|||
uint32 depth = 4;
|
||||
// Bearer token in V2 format.
|
||||
bytes bearer_token = 5;
|
||||
// Result ordering.
|
||||
Order order_by = 6;
|
||||
}
|
||||
|
||||
// Request body.
|
||||
|
|
BIN
pkg/services/tree/service_frostfs.pb.go
generated
BIN
pkg/services/tree/service_frostfs.pb.go
generated
Binary file not shown.
BIN
pkg/services/tree/service_grpc.pb.go
generated
BIN
pkg/services/tree/service_grpc.pb.go
generated
Binary file not shown.
BIN
pkg/services/tree/types.pb.go
generated
BIN
pkg/services/tree/types.pb.go
generated
Binary file not shown.
Loading…
Reference in a new issue