WIP: Morph: Add unit tests #2

Closed
dstepanov-yadro wants to merge 233 commits from TrueCloudLab/frostfs-node:master into object-3608-morph-unit-tests
12 changed files with 81 additions and 22 deletions
Showing only changes of commit b4e72a2dfd - Show all commits

View file

@ -172,7 +172,7 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s
}
// TreeGetChildren implements the pilorama.Forest interface.
func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) {
func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetChildren",
trace.WithAttributes(
attribute.String("container_id", cid.EncodeToString()),
@ -183,7 +183,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree
defer span.End()
var err error
var nodes []uint64
var nodes []pilorama.NodeInfo
for _, sh := range e.sortShardsByWeight(cid) {
nodes, err = sh.TreeGetChildren(ctx, cid, treeID, nodeID)
if err != nil {

View file

@ -927,7 +927,7 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri
}
// TreeGetChildren implements the Forest interface.
func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) {
func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) {
var (
startedAt = time.Now()
success = false
@ -956,7 +956,7 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID
key[0] = 'c'
binary.LittleEndian.PutUint64(key[1:], nodeID)
var children []uint64
var result []NodeInfo
err := t.db.View(func(tx *bbolt.Tx) error {
treeRoot := tx.Bucket(bucketName(cid, treeID))
@ -967,12 +967,23 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID
b := treeRoot.Bucket(dataBucket)
c := b.Cursor()
for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() {
children = append(children, binary.LittleEndian.Uint64(k[9:]))
childID := binary.LittleEndian.Uint64(k[9:])
childInfo := NodeInfo{
ID: childID,
}
parentID, _, metaBytes, found := t.getState(b, stateKey(key, childID))
if found {
childInfo.ParentID = parentID
if err := childInfo.Meta.FromBytes(metaBytes); err != nil {
return err
}
}
result = append(result, childInfo)
}
return nil
})
success = err == nil
return children, metaerr.Wrap(err)
return result, metaerr.Wrap(err)
}
// TreeList implements the Forest interface.

View file

@ -148,7 +148,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string,
}
// TreeGetChildren implements the Forest interface.
func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]uint64, error) {
func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]NodeInfo, error) {
fullID := cid.String() + "/" + treeID
s, ok := f.treeMap[fullID]
if !ok {
@ -156,8 +156,14 @@ func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID str
}
children := s.tree.getChildren(nodeID)
res := make([]Node, len(children))
copy(res, children)
res := make([]NodeInfo, 0, len(children))
for _, childID := range children {
res = append(res, NodeInfo{
ID: childID,
Meta: s.infoMap[childID].Meta,
ParentID: s.infoMap[childID].Parent,
})
}
return res, nil
}

View file

@ -32,7 +32,7 @@ type Forest interface {
TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error)
// TreeGetChildren returns children of the node with the specified ID. The order is arbitrary.
// Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree.
TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error)
TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error)
// TreeGetOpLog returns first log operation stored at or above the height.
// In case no such operation is found, empty Move and nil error should be returned.
TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error)

View file

@ -55,3 +55,9 @@ var (
func isAttributeInternal(key string) bool {
return key == AttributeFilename
}
type NodeInfo struct {
ID Node
Meta Meta
ParentID Node
}

View file

@ -159,7 +159,7 @@ func (s *Shard) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, n
}
// TreeGetChildren implements the pilorama.Forest interface.
func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) {
func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetChildren",
trace.WithAttributes(
attribute.String("shard_id", s.ID().String()),

View file

@ -5,6 +5,7 @@ import (
"context"
"errors"
"fmt"
"sort"
"sync"
"sync/atomic"
@ -440,7 +441,15 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS
func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error {
// Traverse the tree in a DFS manner. Because we need to support arbitrary depth,
// recursive implementation is not suitable here, so we maintain explicit stack.
stack := [][]uint64{{b.GetRootId()}}
m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId())
if err != nil {
return err
}
stack := [][]pilorama.NodeInfo{{{
ID: b.GetRootId(),
Meta: m,
ParentID: p,
}}}
for {
if len(stack) == 0 {
@ -450,19 +459,15 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
continue
}
nodeID := stack[len(stack)-1][0]
node := stack[len(stack)-1][0]
stack[len(stack)-1] = stack[len(stack)-1][1:]
m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), nodeID)
if err != nil {
return err
}
err = srv.Send(&GetSubTreeResponse{
Body: &GetSubTreeResponse_Body{
NodeId: nodeID,
ParentId: p,
Timestamp: m.Time,
Meta: metaToProto(m.Items),
NodeId: node.ID,
ParentId: node.ParentID,
Timestamp: node.Meta.Time,
Meta: metaToProto(node.Meta.Items),
},
})
if err != nil {
@ -470,7 +475,11 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
}
if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() {
children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), nodeID)
children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), node.ID)
if err != nil {
return err
}
children, err = sortByFilename(children, b.GetOrderBy().GetDirection())
if err != nil {
return err
}
@ -482,6 +491,24 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
return nil
}
func sortByFilename(nodes []pilorama.NodeInfo, d GetSubTreeRequest_Body_Order_Direction) ([]pilorama.NodeInfo, error) {
switch d {
case GetSubTreeRequest_Body_Order_None:
return nodes, nil
case GetSubTreeRequest_Body_Order_Asc:
if len(nodes) == 0 {
return nodes, nil
}
less := func(i, j int) bool {
return bytes.Compare(nodes[i].Meta.GetAttr(pilorama.AttributeFilename), nodes[j].Meta.GetAttr(pilorama.AttributeFilename)) < 0
}
sort.Slice(nodes, less)
return nodes, nil
default:
return nil, fmt.Errorf("unsupported order direction: %s", d.String())
}
}
// Apply locally applies operation from the remote node to the tree.
func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) {
err := verifyMessage(req)

Binary file not shown.

View file

@ -238,6 +238,13 @@ message GetNodeByPathResponse {
message GetSubTreeRequest {
message Body {
message Order {
enum Direction {
None = 0;
Asc = 1;
}
Direction direction = 1;
}
// Container ID in V2 format.
bytes container_id = 1;
// The name of the tree.
@ -249,6 +256,8 @@ message GetSubTreeRequest {
uint32 depth = 4;
// Bearer token in V2 format.
bytes bearer_token = 5;
// Result ordering.
Order order_by = 6;
}
// Request body.

Binary file not shown.

Binary file not shown.

Binary file not shown.