WIP: Morph: Add unit tests #2

Closed
dstepanov-yadro wants to merge 233 commits from TrueCloudLab/frostfs-node:master into object-3608-morph-unit-tests
12 changed files with 81 additions and 22 deletions
Showing only changes of commit b4e72a2dfd - Show all commits

View file

@ -172,7 +172,7 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s
} }
// TreeGetChildren implements the pilorama.Forest interface. // TreeGetChildren implements the pilorama.Forest interface.
func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetChildren", ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetChildren",
trace.WithAttributes( trace.WithAttributes(
attribute.String("container_id", cid.EncodeToString()), attribute.String("container_id", cid.EncodeToString()),
@ -183,7 +183,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree
defer span.End() defer span.End()
var err error var err error
var nodes []uint64 var nodes []pilorama.NodeInfo
for _, sh := range e.sortShardsByWeight(cid) { for _, sh := range e.sortShardsByWeight(cid) {
nodes, err = sh.TreeGetChildren(ctx, cid, treeID, nodeID) nodes, err = sh.TreeGetChildren(ctx, cid, treeID, nodeID)
if err != nil { if err != nil {

View file

@ -927,7 +927,7 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri
} }
// TreeGetChildren implements the Forest interface. // TreeGetChildren implements the Forest interface.
func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) { func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) {
var ( var (
startedAt = time.Now() startedAt = time.Now()
success = false success = false
@ -956,7 +956,7 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID
key[0] = 'c' key[0] = 'c'
binary.LittleEndian.PutUint64(key[1:], nodeID) binary.LittleEndian.PutUint64(key[1:], nodeID)
var children []uint64 var result []NodeInfo
err := t.db.View(func(tx *bbolt.Tx) error { err := t.db.View(func(tx *bbolt.Tx) error {
treeRoot := tx.Bucket(bucketName(cid, treeID)) treeRoot := tx.Bucket(bucketName(cid, treeID))
@ -967,12 +967,23 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID
b := treeRoot.Bucket(dataBucket) b := treeRoot.Bucket(dataBucket)
c := b.Cursor() c := b.Cursor()
for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() {
children = append(children, binary.LittleEndian.Uint64(k[9:])) childID := binary.LittleEndian.Uint64(k[9:])
childInfo := NodeInfo{
ID: childID,
}
parentID, _, metaBytes, found := t.getState(b, stateKey(key, childID))
if found {
childInfo.ParentID = parentID
if err := childInfo.Meta.FromBytes(metaBytes); err != nil {
return err
}
}
result = append(result, childInfo)
} }
return nil return nil
}) })
success = err == nil success = err == nil
return children, metaerr.Wrap(err) return result, metaerr.Wrap(err)
} }
// TreeList implements the Forest interface. // TreeList implements the Forest interface.

View file

@ -148,7 +148,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string,
} }
// TreeGetChildren implements the Forest interface. // TreeGetChildren implements the Forest interface.
func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]uint64, error) { func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]NodeInfo, error) {
fullID := cid.String() + "/" + treeID fullID := cid.String() + "/" + treeID
s, ok := f.treeMap[fullID] s, ok := f.treeMap[fullID]
if !ok { if !ok {
@ -156,8 +156,14 @@ func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID str
} }
children := s.tree.getChildren(nodeID) children := s.tree.getChildren(nodeID)
res := make([]Node, len(children)) res := make([]NodeInfo, 0, len(children))
copy(res, children) for _, childID := range children {
res = append(res, NodeInfo{
ID: childID,
Meta: s.infoMap[childID].Meta,
ParentID: s.infoMap[childID].Parent,
})
}
return res, nil return res, nil
} }

View file

@ -32,7 +32,7 @@ type Forest interface {
TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error)
// TreeGetChildren returns children of the node with the specified ID. The order is arbitrary. // TreeGetChildren returns children of the node with the specified ID. The order is arbitrary.
// Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree.
TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error)
// TreeGetOpLog returns first log operation stored at or above the height. // TreeGetOpLog returns first log operation stored at or above the height.
// In case no such operation is found, empty Move and nil error should be returned. // In case no such operation is found, empty Move and nil error should be returned.
TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error)

View file

@ -55,3 +55,9 @@ var (
func isAttributeInternal(key string) bool { func isAttributeInternal(key string) bool {
return key == AttributeFilename return key == AttributeFilename
} }
type NodeInfo struct {
ID Node
Meta Meta
ParentID Node
}

View file

@ -159,7 +159,7 @@ func (s *Shard) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, n
} }
// TreeGetChildren implements the pilorama.Forest interface. // TreeGetChildren implements the pilorama.Forest interface.
func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetChildren", ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetChildren",
trace.WithAttributes( trace.WithAttributes(
attribute.String("shard_id", s.ID().String()), attribute.String("shard_id", s.ID().String()),

View file

@ -5,6 +5,7 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"sort"
"sync" "sync"
"sync/atomic" "sync/atomic"
@ -440,7 +441,15 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS
func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error {
// Traverse the tree in a DFS manner. Because we need to support arbitrary depth, // Traverse the tree in a DFS manner. Because we need to support arbitrary depth,
// recursive implementation is not suitable here, so we maintain explicit stack. // recursive implementation is not suitable here, so we maintain explicit stack.
stack := [][]uint64{{b.GetRootId()}} m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId())
if err != nil {
return err
}
stack := [][]pilorama.NodeInfo{{{
ID: b.GetRootId(),
Meta: m,
ParentID: p,
}}}
for { for {
if len(stack) == 0 { if len(stack) == 0 {
@ -450,19 +459,15 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
continue continue
} }
nodeID := stack[len(stack)-1][0] node := stack[len(stack)-1][0]
stack[len(stack)-1] = stack[len(stack)-1][1:] stack[len(stack)-1] = stack[len(stack)-1][1:]
m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), nodeID)
if err != nil {
return err
}
err = srv.Send(&GetSubTreeResponse{ err = srv.Send(&GetSubTreeResponse{
Body: &GetSubTreeResponse_Body{ Body: &GetSubTreeResponse_Body{
NodeId: nodeID, NodeId: node.ID,
ParentId: p, ParentId: node.ParentID,
Timestamp: m.Time, Timestamp: node.Meta.Time,
Meta: metaToProto(m.Items), Meta: metaToProto(node.Meta.Items),
}, },
}) })
if err != nil { if err != nil {
@ -470,7 +475,11 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
} }
if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() { if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() {
children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), nodeID) children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), node.ID)
if err != nil {
return err
}
children, err = sortByFilename(children, b.GetOrderBy().GetDirection())
if err != nil { if err != nil {
return err return err
} }
@ -482,6 +491,24 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
return nil return nil
} }
func sortByFilename(nodes []pilorama.NodeInfo, d GetSubTreeRequest_Body_Order_Direction) ([]pilorama.NodeInfo, error) {
switch d {
case GetSubTreeRequest_Body_Order_None:
return nodes, nil
case GetSubTreeRequest_Body_Order_Asc:
if len(nodes) == 0 {
return nodes, nil
}
less := func(i, j int) bool {
return bytes.Compare(nodes[i].Meta.GetAttr(pilorama.AttributeFilename), nodes[j].Meta.GetAttr(pilorama.AttributeFilename)) < 0
}
sort.Slice(nodes, less)
return nodes, nil
default:
return nil, fmt.Errorf("unsupported order direction: %s", d.String())
}
}
// Apply locally applies operation from the remote node to the tree. // Apply locally applies operation from the remote node to the tree.
func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) { func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) {
err := verifyMessage(req) err := verifyMessage(req)

Binary file not shown.

View file

@ -238,6 +238,13 @@ message GetNodeByPathResponse {
message GetSubTreeRequest { message GetSubTreeRequest {
message Body { message Body {
message Order {
enum Direction {
None = 0;
Asc = 1;
}
Direction direction = 1;
}
// Container ID in V2 format. // Container ID in V2 format.
bytes container_id = 1; bytes container_id = 1;
// The name of the tree. // The name of the tree.
@ -249,6 +256,8 @@ message GetSubTreeRequest {
uint32 depth = 4; uint32 depth = 4;
// Bearer token in V2 format. // Bearer token in V2 format.
bytes bearer_token = 5; bytes bearer_token = 5;
// Result ordering.
Order order_by = 6;
} }
// Request body. // Request body.

Binary file not shown.

Binary file not shown.

Binary file not shown.