From 9d1c915c428a4d07bc2bb62db2fe1832dfaecdb3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 Jul 2024 09:30:01 +0300 Subject: [PATCH] [#1251] pilorama: Allow traversing multiple branches in parallel Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/tree/move.go | 2 +- cmd/frostfs-cli/modules/tree/subtree.go | 15 ++- pkg/local_object_storage/engine/tree.go | 5 +- pkg/local_object_storage/pilorama/boltdb.go | 120 ++++++++++++------ pkg/local_object_storage/pilorama/forest.go | 49 +++---- .../pilorama/forest_test.go | 34 ++--- pkg/local_object_storage/pilorama/heap.go | 4 +- .../pilorama/interface.go | 2 +- pkg/local_object_storage/pilorama/meta.go | 6 +- .../pilorama/multinode.go | 49 +++++++ .../pilorama/split_test.go | 57 +++++++++ pkg/local_object_storage/shard/tree.go | 3 +- pkg/services/tree/getsubtree_test.go | 35 +++-- pkg/services/tree/service.go | 92 +++++++++----- pkg/services/tree/service.pb.go | Bin 124265 -> 123722 bytes pkg/services/tree/service.proto | 10 +- pkg/services/tree/service_frostfs.pb.go | Bin 59468 -> 60583 bytes pkg/services/tree/types.pb.go | Bin 9911 -> 9855 bytes 18 files changed, 342 insertions(+), 141 deletions(-) create mode 100644 pkg/local_object_storage/pilorama/multinode.go diff --git a/cmd/frostfs-cli/modules/tree/move.go b/cmd/frostfs-cli/modules/tree/move.go index 84b2fb80e..95516940c 100644 --- a/cmd/frostfs-cli/modules/tree/move.go +++ b/cmd/frostfs-cli/modules/tree/move.go @@ -66,7 +66,7 @@ func move(cmd *cobra.Command, _ []string) { Body: &tree.GetSubTreeRequest_Body{ ContainerId: rawCID, TreeId: tid, - RootId: nid, + RootId: []uint64{nid}, Depth: 1, BearerToken: bt, }, diff --git a/cmd/frostfs-cli/modules/tree/subtree.go b/cmd/frostfs-cli/modules/tree/subtree.go index 64cb351ec..e58a13fd6 100644 --- a/cmd/frostfs-cli/modules/tree/subtree.go +++ b/cmd/frostfs-cli/modules/tree/subtree.go @@ -68,7 +68,7 @@ func getSubTree(cmd *cobra.Command, _ []string) { Body: &tree.GetSubTreeRequest_Body{ ContainerId: rawCID, TreeId: tid, - RootId: rid, + RootId: []uint64{rid}, Depth: depth, BearerToken: bt, }, @@ -83,10 +83,15 @@ func getSubTree(cmd *cobra.Command, _ []string) { for ; err == nil; subtreeResp, err = resp.Recv() { b := subtreeResp.GetBody() - cmd.Printf("Node ID: %d\n", b.GetNodeId()) - - cmd.Println("\tParent ID: ", b.GetParentId()) - cmd.Println("\tTimestamp: ", b.GetTimestamp()) + if len(b.GetNodeId()) == 1 { + cmd.Printf("Node ID: %d\n", b.GetNodeId()) + cmd.Println("\tParent ID: ", b.GetParentId()) + cmd.Println("\tTimestamp: ", b.GetTimestamp()) + } else { + cmd.Printf("Node IDs: %v\n", b.GetNodeId()) + cmd.Println("\tParent IDs: ", b.GetParentId()) + cmd.Println("\tTimestamps: ", b.GetTimestamp()) + } if meta := b.GetMeta(); len(meta) > 0 { cmd.Println("\tMeta pairs: ") diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 8bacdba76..39122628f 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -210,18 +210,17 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree } // TreeSortedByFilename implements the pilorama.Forest interface. -func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node, last *string, count int) ([]pilorama.NodeInfo, *string, error) { +func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *string, count int) ([]pilorama.MultiNodeInfo, *string, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeSortedByFilename", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("node_id", strconv.FormatUint(nodeID, 10)), ), ) defer span.End() var err error - var nodes []pilorama.NodeInfo + var nodes []pilorama.MultiNodeInfo var cursor *string for _, sh := range e.sortShards(cid) { nodes, cursor, err = sh.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 4f09fe4cd..a98d8b2b9 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -990,23 +990,26 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri return m, parentID, metaerr.Wrap(err) } -func (t *boltForest) hasFewChildren(b *bbolt.Bucket, nodeID Node, threshold int) bool { +func (t *boltForest) hasFewChildren(b *bbolt.Bucket, nodeIDs MultiNode, threshold int) bool { key := make([]byte, 9) key[0] = 'c' - binary.LittleEndian.PutUint64(key[1:], nodeID) count := 0 - c := b.Cursor() - for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { - if count++; count > threshold { - return false + for _, nodeID := range nodeIDs { + binary.LittleEndian.PutUint64(key[1:], nodeID) + + c := b.Cursor() + for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { + if count++; count > threshold { + return false + } } } return true } // TreeSortedByFilename implements the Forest interface. -func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node, last *string, count int) ([]NodeInfo, *string, error) { +func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeIDs MultiNode, last *string, count int) ([]MultiNodeInfo, *string, error) { var ( startedAt = time.Now() success = false @@ -1019,7 +1022,6 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("node_id", strconv.FormatUint(nodeID, 10)), ), ) defer span.End() @@ -1030,6 +1032,9 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr if t.mode.NoMetabase() { return nil, last, ErrDegradedMode } + if len(nodeIDs) == 0 { + return nil, last, errors.New("empty node list") + } h := newHeap(last, count) key := make([]byte, 9) @@ -1048,20 +1053,22 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr // If the node is a leaf, we could scan all filenames in the tree. // To prevent this we first count the number of children: if it is less than // the number of nodes we need to return, fallback to TreeGetChildren() implementation. - if fewChildren = t.hasFewChildren(b, nodeID, count); fewChildren { + if fewChildren = t.hasFewChildren(b, nodeIDs, count); fewChildren { var err error - result, err = t.getChildren(b, nodeID) + result, err = t.getChildren(b, nodeIDs) return err } - t.fillSortedChildren(b, nodeID, h) + t.fillSortedChildren(b, nodeIDs, h) for info, ok := h.pop(); ok; info, ok = h.pop() { - childInfo, err := t.getChildInfo(b, key, info.id) - if err != nil { - return err + for _, id := range info.id { + childInfo, err := t.getChildInfo(b, key, id) + if err != nil { + return err + } + result = append(result, childInfo) } - result = append(result, childInfo) } return nil }) @@ -1074,11 +1081,15 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr if fewChildren { result = sortAndCut(result, last) } - if len(result) != 0 { - s := string(result[len(result)-1].Meta.GetAttr(AttributeFilename)) + res := mergeNodeInfos(result) + if len(res) > count { + res = res[:count] + } + if len(res) != 0 { + s := string(findAttr(res[len(res)-1].Meta, AttributeFilename)) last = &s } - return result, last, metaerr.Wrap(err) + return res, last, metaerr.Wrap(err) } func sortAndCut(result []NodeInfo, last *string) []NodeInfo { @@ -1109,37 +1120,64 @@ func (t *boltForest) getChildInfo(b *bbolt.Bucket, key []byte, childID Node) (No return childInfo, nil } -func (t *boltForest) fillSortedChildren(b *bbolt.Bucket, nodeID Node, h *fixedHeap) { +func (t *boltForest) fillSortedChildren(b *bbolt.Bucket, nodeIDs MultiNode, h *fixedHeap) { c := b.Cursor() prefix := internalKeyPrefix(nil, AttributeFilename) length := uint16(0) count := 0 + + var nodes []uint64 + var lastFilename *string for k, _ := c.Seek(prefix); len(k) > 0 && k[0] == 'i'; k, _ = c.Next() { if len(k) < len(prefix)+2+16 { continue } parentID := binary.LittleEndian.Uint64(k[len(k)-16:]) - if parentID != nodeID { + + var contains bool + for i := range nodeIDs { + if parentID == nodeIDs[i] { + contains = true + break + } + } + if !contains { continue } actualLength := binary.LittleEndian.Uint16(k[len(prefix):]) childID := binary.LittleEndian.Uint64(k[len(k)-8:]) filename := string(k[len(prefix)+2 : len(k)-16]) - processed := h.push(childID, filename) - if actualLength != length { - length = actualLength - count = 1 - } else if processed { - if count++; count > h.count { - length = actualLength + 1 - c.Seek(append(prefix, byte(length), byte(length>>8))) - c.Prev() // c.Next() will be performed by for loop + + if lastFilename == nil { + lastFilename = &filename + nodes = append(nodes, childID) + } else if *lastFilename == filename { + nodes = append(nodes, childID) + } else { + processed := h.push(nodes, *lastFilename) + nodes = MultiNode{childID} + lastFilename = &filename + if actualLength != length { + length = actualLength + count = 1 + } else if processed { + if count++; count > h.count { + lastFilename = nil + nodes = nil + length = actualLength + 1 + c.Seek(append(prefix, byte(length), byte(length>>8))) + c.Prev() // c.Next() will be performed by for loop + } } } } + + if len(nodes) != 0 && lastFilename != nil { + h.push(nodes, *lastFilename) + } } // TreeGetChildren implements the Forest interface. @@ -1179,28 +1217,30 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID b := treeRoot.Bucket(dataBucket) var err error - result, err = t.getChildren(b, nodeID) + result, err = t.getChildren(b, []Node{nodeID}) return err }) success = err == nil return result, metaerr.Wrap(err) } -func (t *boltForest) getChildren(b *bbolt.Bucket, nodeID Node) ([]NodeInfo, error) { +func (t *boltForest) getChildren(b *bbolt.Bucket, nodeIDs MultiNode) ([]NodeInfo, error) { var result []NodeInfo key := make([]byte, 9) - key[0] = 'c' - binary.LittleEndian.PutUint64(key[1:], nodeID) + for _, nodeID := range nodeIDs { + key[0] = 'c' + binary.LittleEndian.PutUint64(key[1:], nodeID) - c := b.Cursor() - for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { - childID := binary.LittleEndian.Uint64(k[9:]) - childInfo, err := t.getChildInfo(b, key, childID) - if err != nil { - return nil, err + c := b.Cursor() + for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { + childID := binary.LittleEndian.Uint64(k[9:]) + childInfo, err := t.getChildInfo(b, key, childID) + if err != nil { + return nil, err + } + result = append(result, childInfo) } - result = append(result, childInfo) } return result, nil } diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index b45a77b99..78503bada 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -156,7 +156,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string, } // TreeSortedByFilename implements the Forest interface. -func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeID Node, start *string, count int) ([]NodeInfo, *string, error) { +func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeIDs MultiNode, start *string, count int) ([]MultiNodeInfo, *string, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -166,40 +166,45 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI return nil, start, nil } - children := s.tree.getChildren(nodeID) - res := make([]NodeInfo, 0, len(children)) - for _, childID := range children { - var found bool - for _, kv := range s.infoMap[childID].Meta.Items { - if kv.Key == AttributeFilename { - found = true - break + var res []NodeInfo + + for _, nodeID := range nodeIDs { + children := s.tree.getChildren(nodeID) + for _, childID := range children { + var found bool + for _, kv := range s.infoMap[childID].Meta.Items { + if kv.Key == AttributeFilename { + found = true + break + } } + if !found { + continue + } + res = append(res, NodeInfo{ + ID: childID, + Meta: s.infoMap[childID].Meta, + ParentID: s.infoMap[childID].Parent, + }) } - if !found { - continue - } - res = append(res, NodeInfo{ - ID: childID, - Meta: s.infoMap[childID].Meta, - ParentID: s.infoMap[childID].Parent, - }) } if len(res) == 0 { - return res, start, nil + return nil, start, nil } sort.Slice(res, func(i, j int) bool { return bytes.Compare(res[i].Meta.GetAttr(AttributeFilename), res[j].Meta.GetAttr(AttributeFilename)) == -1 }) - for i := range res { - if start == nil || string(res[i].Meta.GetAttr(AttributeFilename)) > *start { + + r := mergeNodeInfos(res) + for i := range r { + if start == nil || string(findAttr(r[i].Meta, AttributeFilename)) > *start { finish := i + count if len(res) < finish { finish = len(res) } - last := string(res[finish-1].Meta.GetAttr(AttributeFilename)) - return res[i:finish], &last, nil + last := string(findAttr(r[finish-1].Meta, AttributeFilename)) + return r[i:finish], &last, nil } } last := string(res[len(res)-1].Meta.GetAttr(AttributeFilename)) diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 9da0177cc..001d095c8 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -215,7 +215,7 @@ func BenchmarkForestSortedIteration(b *testing.B) { b.Run(providers[i].name+",root", func(b *testing.B) { for i := 0; i < b.N; i++ { - res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, RootID, nil, 100) + res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, MultiNode{RootID}, nil, 100) if err != nil || len(res) != 100 { b.Fatalf("err %v, count %d", err, len(res)) } @@ -223,7 +223,7 @@ func BenchmarkForestSortedIteration(b *testing.B) { }) b.Run(providers[i].name+",leaf", func(b *testing.B) { for i := 0; i < b.N; i++ { - res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, 1, nil, 100) + res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, MultiNode{1}, nil, 100) if err != nil || len(res) != 0 { b.FailNow() } @@ -266,9 +266,9 @@ func testForestTreeSortedIteration(t *testing.T, s ForestStorage) { treeAdd(t, i+1, strconv.Itoa(i+1)) } - var result []NodeInfo + var result []MultiNodeInfo treeAppend := func(t *testing.T, last *string, count int) *string { - res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, RootID, last, count) + res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, last, count) require.NoError(t, err) result = append(result, res...) return cursor @@ -282,11 +282,11 @@ func testForestTreeSortedIteration(t *testing.T, s ForestStorage) { require.Len(t, result, count) for i := range result { - require.Equal(t, RootID+uint64(i+1), result[i].ID) + require.Equal(t, MultiNode{RootID + uint64(i+1)}, result[i].Children) if i == 0 { - require.Equal(t, "", string(result[i].Meta.GetAttr(AttributeFilename))) + require.Equal(t, "", string(findAttr(result[i].Meta, AttributeFilename))) } else { - require.Equal(t, strconv.Itoa(RootID+i+1), string(result[i].Meta.GetAttr(AttributeFilename))) + require.Equal(t, strconv.Itoa(RootID+i+1), string(findAttr(result[i].Meta, AttributeFilename))) } } } @@ -318,12 +318,12 @@ func testForestTreeSortedByFilename(t *testing.T, s ForestStorage) { require.NoError(t, err) } - expectAttributes := func(t *testing.T, attr string, expected []string, res []NodeInfo) { + expectAttributes := func(t *testing.T, attr string, expected []string, res []MultiNodeInfo) { require.Equal(t, len(expected), len(res)) actual := make([]string, len(res)) for i := range actual { - actual[i] = string(res[i].Meta.GetAttr(attr)) + actual[i] = string(findAttr(res[i].Meta, attr)) } require.Equal(t, expected, actual) } @@ -345,40 +345,40 @@ func testForestTreeSortedByFilename(t *testing.T, s ForestStorage) { treeAddByPath(t, items[i]) } - getChildren := func(t *testing.T, id Node) []NodeInfo { + getChildren := func(t *testing.T, id MultiNode) []MultiNodeInfo { res, _, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, id, nil, len(items)) require.NoError(t, err) return res } - res := getChildren(t, RootID) + res := getChildren(t, MultiNode{RootID}) expectAttributes(t, AttributeFilename, []string{"a", "b", "c"}, res) expectAttributes(t, controlAttr, []string{"", "", "c"}, res) { - ra := getChildren(t, res[0].ID) + ra := getChildren(t, res[0].Children) expectAttributes(t, AttributeFilename, []string{"bbb"}, ra) expectAttributes(t, controlAttr, []string{""}, ra) - rabbb := getChildren(t, ra[0].ID) + rabbb := getChildren(t, ra[0].Children) expectAttributes(t, AttributeFilename, []string{"ccc", "xxx", "z"}, rabbb) expectAttributes(t, controlAttr, []string{"a/bbb/ccc", "a/bbb/xxx", "a/bbb/z"}, rabbb) } { - rb := getChildren(t, res[1].ID) + rb := getChildren(t, res[1].Children) expectAttributes(t, AttributeFilename, []string{"bbb", "xxx"}, rb) expectAttributes(t, controlAttr, []string{"", ""}, rb) - rbbbb := getChildren(t, rb[0].ID) + rbbbb := getChildren(t, rb[0].Children) expectAttributes(t, AttributeFilename, []string{"ccc"}, rbbbb) expectAttributes(t, controlAttr, []string{"b/bbb/ccc"}, rbbbb) - rbxxx := getChildren(t, rb[1].ID) + rbxxx := getChildren(t, rb[1].Children) expectAttributes(t, AttributeFilename, []string{"z"}, rbxxx) expectAttributes(t, controlAttr, []string{"b/xxx/z"}, rbxxx) } { - rc := getChildren(t, res[2].ID) + rc := getChildren(t, res[2].Children) require.Len(t, rc, 0) } } diff --git a/pkg/local_object_storage/pilorama/heap.go b/pkg/local_object_storage/pilorama/heap.go index b40fb0849..362b92cfb 100644 --- a/pkg/local_object_storage/pilorama/heap.go +++ b/pkg/local_object_storage/pilorama/heap.go @@ -5,7 +5,7 @@ import ( ) type heapInfo struct { - id Node + id MultiNode filename string } @@ -46,7 +46,7 @@ func newHeap(start *string, count int) *fixedHeap { } } -func (h *fixedHeap) push(id Node, filename string) bool { +func (h *fixedHeap) push(id MultiNode, filename string) bool { if h.start != nil && filename <= *h.start { return false } diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index f7f5a85b1..61a3849bf 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -35,7 +35,7 @@ type Forest interface { TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) // TreeSortedByFilename returns children of the node with the specified ID. The nodes are sorted by the filename attribute.. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. - TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node, last *string, count int) ([]NodeInfo, *string, error) + TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID MultiNode, last *string, count int) ([]MultiNodeInfo, *string, error) // TreeGetOpLog returns first log operation stored at or above the height. // In case no such operation is found, empty Move and nil error should be returned. TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) diff --git a/pkg/local_object_storage/pilorama/meta.go b/pkg/local_object_storage/pilorama/meta.go index 49b7f6477..45e9c2f79 100644 --- a/pkg/local_object_storage/pilorama/meta.go +++ b/pkg/local_object_storage/pilorama/meta.go @@ -21,7 +21,11 @@ func (x Meta) Bytes() []byte { } func (x Meta) GetAttr(name string) []byte { - for _, kv := range x.Items { + return findAttr(x.Items, name) +} + +func findAttr(ms []KeyValue, name string) []byte { + for _, kv := range ms { if kv.Key == name { return kv.Value } diff --git a/pkg/local_object_storage/pilorama/multinode.go b/pkg/local_object_storage/pilorama/multinode.go new file mode 100644 index 000000000..106ba6ae9 --- /dev/null +++ b/pkg/local_object_storage/pilorama/multinode.go @@ -0,0 +1,49 @@ +package pilorama + +import "bytes" + +// MultiNode represents a group of internal nodes accessible by the same path, but having different id. +type MultiNode []Node + +// MultiNodeInfo represents a group of internal nodes accessible by the same path, but having different id. +type MultiNodeInfo struct { + Children MultiNode + Parents MultiNode + Timestamps []uint64 + Meta []KeyValue +} + +func (r *MultiNodeInfo) Add(info NodeInfo) bool { + if !isInternal(info.Meta.Items) || !isInternal(r.Meta) || + !bytes.Equal(r.Meta[0].Value, info.Meta.Items[0].Value) { + return false + } + + r.Children = append(r.Children, info.ID) + r.Parents = append(r.Parents, info.ParentID) + r.Timestamps = append(r.Timestamps, info.Meta.Time) + return true +} + +func (n NodeInfo) ToMultiNode() MultiNodeInfo { + return MultiNodeInfo{ + Children: MultiNode{n.ID}, + Parents: MultiNode{n.ParentID}, + Timestamps: []uint64{n.Meta.Time}, + Meta: n.Meta.Items, + } +} + +func isInternal(m []KeyValue) bool { + return len(m) == 1 && m[0].Key == AttributeFilename +} + +func mergeNodeInfos(ns []NodeInfo) []MultiNodeInfo { + var r []MultiNodeInfo + for _, info := range ns { + if len(r) == 0 || !r[len(r)-1].Add(info) { + r = append(r, info.ToMultiNode()) + } + } + return r +} diff --git a/pkg/local_object_storage/pilorama/split_test.go b/pkg/local_object_storage/pilorama/split_test.go index 3bd581ce6..54c2b90a6 100644 --- a/pkg/local_object_storage/pilorama/split_test.go +++ b/pkg/local_object_storage/pilorama/split_test.go @@ -95,4 +95,61 @@ func testDuplicateDirectory(t *testing.T, f Forest) { require.Equal(t, []byte{4}, testGetByPath(t, "dir1/dir3/value3")) require.Equal(t, []byte{8}, testGetByPath(t, "dir1/dir3/value4")) require.Equal(t, []byte{10}, testGetByPath(t, "value0")) + + testSortedByFilename := func(t *testing.T, root MultiNode, last *string, batchSize int) ([]MultiNodeInfo, *string) { + res, last, err := f.TreeSortedByFilename(context.Background(), d.CID, treeID, root, last, batchSize) + require.NoError(t, err) + return res, last + } + + t.Run("test sorted listing, full children branch", func(t *testing.T) { + t.Run("big batch size", func(t *testing.T) { + res, _ := testSortedByFilename(t, MultiNode{RootID}, nil, 10) + require.Equal(t, 3, len(res)) + require.Equal(t, MultiNode{1, 5}, res[0].Children) + require.Equal(t, MultiNode{9}, res[1].Children) + require.Equal(t, MultiNode{10}, res[2].Children) + + t.Run("multi-root", func(t *testing.T) { + res, _ := testSortedByFilename(t, MultiNode{1, 5}, nil, 10) + require.Equal(t, 3, len(res)) + require.Equal(t, MultiNode{3, 7}, res[0].Children) + require.Equal(t, MultiNode{2}, res[1].Children) + require.Equal(t, MultiNode{6}, res[2].Children) + }) + }) + t.Run("small batch size", func(t *testing.T) { + res, last := testSortedByFilename(t, MultiNode{RootID}, nil, 1) + require.Equal(t, 1, len(res)) + require.Equal(t, MultiNode{1, 5}, res[0].Children) + + res, last = testSortedByFilename(t, MultiNode{RootID}, last, 1) + require.Equal(t, 1, len(res)) + require.Equal(t, MultiNode{9}, res[0].Children) + + res, last = testSortedByFilename(t, MultiNode{RootID}, last, 1) + require.Equal(t, 1, len(res)) + require.Equal(t, MultiNode{10}, res[0].Children) + + res, _ = testSortedByFilename(t, MultiNode{RootID}, last, 1) + require.Equal(t, 0, len(res)) + + t.Run("multi-root", func(t *testing.T) { + res, last := testSortedByFilename(t, MultiNode{1, 5}, nil, 1) + require.Equal(t, 1, len(res)) + require.Equal(t, MultiNode{3, 7}, res[0].Children) + + res, last = testSortedByFilename(t, MultiNode{1, 5}, last, 1) + require.Equal(t, 1, len(res)) + require.Equal(t, MultiNode{2}, res[0].Children) + + res, last = testSortedByFilename(t, MultiNode{1, 5}, last, 1) + require.Equal(t, 1, len(res)) + require.Equal(t, MultiNode{6}, res[0].Children) + + res, _ = testSortedByFilename(t, MultiNode{RootID}, last, 1) + require.Equal(t, 0, len(res)) + }) + }) + }) } diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index e92a61e5b..26dc8ec1e 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -184,13 +184,12 @@ func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID strin } // TreeSortedByFilename implements the pilorama.Forest interface. -func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node, last *string, count int) ([]pilorama.NodeInfo, *string, error) { +func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *string, count int) ([]pilorama.MultiNodeInfo, *string, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeSortedByFilename", trace.WithAttributes( attribute.String("shard_id", s.ID().String()), attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("node_id", strconv.FormatUint(nodeID, 10)), ), ) defer span.End() diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index 9a4223e30..7c109f538 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -48,7 +48,7 @@ func TestGetSubTree(t *testing.T) { acc := subTreeAcc{errIndex: errIndex} err := getSubTree(context.Background(), &acc, d.CID, &GetSubTreeRequest_Body{ TreeId: treeID, - RootId: rootID, + RootId: []uint64{rootID}, Depth: depth, }, p) if errIndex == -1 { @@ -58,12 +58,12 @@ func TestGetSubTree(t *testing.T) { } // GetSubTree must return child only after is has returned the parent. - require.Equal(t, rootID, acc.seen[0].Body.NodeId) + require.Equal(t, rootID, acc.seen[0].Body.NodeId[0]) loop: for i := 1; i < len(acc.seen); i++ { parent := acc.seen[i].Body.ParentId for j := 0; j < i; j++ { - if acc.seen[j].Body.NodeId == parent { + if acc.seen[j].Body.NodeId[0] == parent[0] { continue loop } } @@ -73,16 +73,16 @@ func TestGetSubTree(t *testing.T) { // GetSubTree must return valid meta. for i := range acc.seen { b := acc.seen[i].Body - meta, node, err := p.TreeGetMeta(context.Background(), d.CID, treeID, b.NodeId) + meta, node, err := p.TreeGetMeta(context.Background(), d.CID, treeID, b.NodeId[0]) require.NoError(t, err) - require.Equal(t, node, b.ParentId) - require.Equal(t, meta.Time, b.Timestamp) + require.Equal(t, node, b.ParentId[0]) + require.Equal(t, meta.Time, b.Timestamp[0]) require.Equal(t, metaToProto(meta.Items), b.Meta) } ordered := make([]uint64, len(acc.seen)) for i := range acc.seen { - ordered[i] = acc.seen[i].Body.NodeId + ordered[i] = acc.seen[i].Body.NodeId[0] } return ordered } @@ -184,7 +184,7 @@ func testGetSubTreeOrderAsc(t *testing.T, p pilorama.ForestStorage) { } found := false for j := range tree { - if acc.seen[i].Body.NodeId == tree[j].id { + if acc.seen[i].Body.NodeId[0] == tree[j].id { found = true paths = append(paths, path.Join(tree[j].path...)) } @@ -207,7 +207,7 @@ func testGetSubTreeOrderAsc(t *testing.T, p pilorama.ForestStorage) { }, p) require.NoError(t, err) require.Len(t, acc.seen, 1) - require.Equal(t, uint64(0), acc.seen[0].Body.NodeId) + require.Equal(t, uint64(0), acc.seen[0].Body.NodeId[0]) }) t.Run("depth=2", func(t *testing.T) { acc := subTreeAcc{errIndex: -1} @@ -220,15 +220,16 @@ func testGetSubTreeOrderAsc(t *testing.T, p pilorama.ForestStorage) { }, p) require.NoError(t, err) require.Len(t, acc.seen, 3) - require.Equal(t, uint64(0), acc.seen[0].Body.NodeId) - require.Equal(t, uint64(0), acc.seen[1].GetBody().GetParentId()) - require.Equal(t, uint64(0), acc.seen[2].GetBody().GetParentId()) + require.Equal(t, uint64(0), acc.seen[0].Body.NodeId[0]) + require.Equal(t, uint64(0), acc.seen[1].GetBody().GetParentId()[0]) + require.Equal(t, uint64(0), acc.seen[2].GetBody().GetParentId()[0]) }) } var ( errSubTreeSend = errors.New("send finished with error") errSubTreeSendAfterError = errors.New("send was invoked after an error occurred") + errInvalidResponse = errors.New("send got invalid response") ) type subTreeAcc struct { @@ -241,6 +242,16 @@ type subTreeAcc struct { var _ TreeService_GetSubTreeServer = &subTreeAcc{} func (s *subTreeAcc) Send(r *GetSubTreeResponse) error { + b := r.GetBody() + if len(b.GetNodeId()) > 1 { + return errInvalidResponse + } + if len(b.GetParentId()) > 1 { + return errInvalidResponse + } + if len(b.GetTimestamp()) > 1 { + return errInvalidResponse + } s.seen = append(s.seen, r) if s.errIndex >= 0 { if len(s.seen) == s.errIndex+1 { diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 6d3e24432..b74f6d2b4 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -16,6 +16,8 @@ import ( netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/panjf2000/ants/v2" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // Service represents tree-service capable of working with multiple @@ -440,29 +442,50 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS return getSubTree(srv.Context(), srv, cid, b, s.forest) } +type stackItem struct { + values []pilorama.MultiNodeInfo + parent pilorama.MultiNode + last *string +} + func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { const batchSize = 1000 - type stackItem struct { - values []pilorama.NodeInfo - parent pilorama.Node - last *string + // For backward compatibility. + rootIDs := b.GetRootId() + if len(rootIDs) == 0 { + rootIDs = []uint64{0} } // Traverse the tree in a DFS manner. Because we need to support arbitrary depth, // recursive implementation is not suitable here, so we maintain explicit stack. - m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId()) - if err != nil { - return err + var ms []pilorama.KeyValue + var ps []uint64 + var ts []uint64 + for _, rootID := range rootIDs { + m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), rootID) + if err != nil { + return err + } + if ms == nil { + ms = m.Items + } else { + if len(m.Items) != 1 { + return status.Error(codes.InvalidArgument, "multiple non-internal nodes provided") + } + } + ts = append(ts, m.Time) + ps = append(ps, p) } stack := []stackItem{{ - values: []pilorama.NodeInfo{{ - ID: b.GetRootId(), - Meta: m, - ParentID: p, + values: []pilorama.MultiNodeInfo{{ + Children: rootIDs, + Timestamps: ts, + Meta: ms, + Parents: ps, }}, - parent: p, + parent: ps, }} for { @@ -486,30 +509,20 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid } } - node := stack[len(stack)-1].values[0] - stack[len(stack)-1].values = stack[len(stack)-1].values[1:] - - err = srv.Send(&GetSubTreeResponse{ - Body: &GetSubTreeResponse_Body{ - NodeId: node.ID, - ParentId: node.ParentID, - Timestamp: node.Meta.Time, - Meta: metaToProto(node.Meta.Items), - }, - }) + node, err := stackPopAndSend(stack, srv) if err != nil { return err } if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() { - children, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), node.ID, nil, batchSize) + children, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), node.Children, nil, batchSize) if err != nil { return err } if len(children) != 0 { stack = append(stack, stackItem{ values: children, - parent: node.ID, + parent: node.Children, last: last, }) } @@ -518,19 +531,38 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid return nil } +func stackPopAndSend(stack []stackItem, srv TreeService_GetSubTreeServer) (pilorama.MultiNodeInfo, error) { + node := stack[len(stack)-1].values[0] + stack[len(stack)-1].values = stack[len(stack)-1].values[1:] + + return node, srv.Send(&GetSubTreeResponse{ + Body: &GetSubTreeResponse_Body{ + NodeId: node.Children, + ParentId: node.Parents, + Timestamp: node.Timestamps, + Meta: metaToProto(node.Meta), + }, + }) +} + func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { if b.GetOrderBy().GetDirection() == GetSubTreeRequest_Body_Order_Asc { return getSortedSubTree(ctx, srv, cid, b, forest) } + var rootID uint64 + if len(b.GetRootId()) > 0 { + rootID = b.GetRootId()[0] + } + // Traverse the tree in a DFS manner. Because we need to support arbitrary depth, // recursive implementation is not suitable here, so we maintain explicit stack. - m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId()) + m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), rootID) if err != nil { return err } stack := [][]pilorama.NodeInfo{{{ - ID: b.GetRootId(), + ID: rootID, Meta: m, ParentID: p, }}} @@ -548,9 +580,9 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD err = srv.Send(&GetSubTreeResponse{ Body: &GetSubTreeResponse_Body{ - NodeId: node.ID, - ParentId: node.ParentID, - Timestamp: node.Meta.Time, + NodeId: []uint64{node.ID}, + ParentId: []uint64{node.ParentID}, + Timestamp: []uint64{node.Meta.Time}, Meta: metaToProto(node.Meta.Items), }, }) diff --git a/pkg/services/tree/service.pb.go b/pkg/services/tree/service.pb.go index 7930361e171ebdb643101e500727e6105aeb32e9..2ca3c0528763e90d2cead13c1bb9709d70986252 100644 GIT binary patch delta 3910 zcmai0YfzNu6`lhu_W%-H2na56S0cO1Z83=AXuzlu!o?s`Sh*;aa1#ht1PllUn@Hws zI1}0?blU0IW*V_HZZ1Tr8cfg%wvDvb3eHp;lUlKVI!!e6dEXCc9X{GW-|Y9k=RD^* z=RD_pduP;Xbi}Er%|33m`DFXsW3l@T)n$hIy`_e_>gsw!Rdq?JJh2+~HSDghD=jsY zRo9j7t6%Nv@*E5{ml*O3l+3eF=whSmN^1t|$3Jnj%P-(V_7tKNPIxPY?Ow{j)qugi zi4iAUNK=zARaz!@sRse};`PAm8%Qd7(;1Y;TOw&m zNSs-9i*|J}Hkpj_MLy%zi-c$cXJt~Fqqt8ra&jt#aODbe=dnzn52sKd2U^I?Z$Cla zcCIem7D7w8IhuTUCXQVBbSSL0TgcZQcs%hOr3s_uS>(%8{^ZF|XHgP&ZKD@?AcsQP zKbs=_MJSb5Vv-bS~+sPdA@fE=m;$;BURl>w^e>VAZS1!Ok@DQXMbAgy& zPAXC()y%bc&He2%i^%D7cmmWTAM;CRAi#CwWsDAB!ZD zC`@WP1@O!sU>O ze_MdM>60)+|G#A>i@;vZ?-oPE>2*jErE%w>^?2w?$Rdz`)Cm#pDxnP=w+5k$k|A2m zl^0Kzpc?PAVGCS643oSPTFFyp^0c*u(+*-=85HcX5iZBJQwkSF1M^r0>9}eqq!WyJ z9*9%txwZy{cxxR>G8u_#oY@T?Z}}q3crdNt?h-f_?ImYtA&*3~lxy}^7Lh=}(x!fu%kQ**sZ8fEo zMm+Ovij}d97$#m*iC6`lpkz!2`JYwN7I#5J-2x^n-}gu5C);7Lvr=}5X)Qga?tmga zmLmGRnfQw;%CN;m(^lfn8nv?hdm-wK1aj|91i!mAWEO<0Yf&;wEv2jZVsG$~T5x!+ z9H`6cphav%9sjc&IQV? zTD>Tm4+dS=K%4l62}3Jt$Ypqy8P4vESU%B+nVD`vsF_9>JhajZS<$#gOa~wWxw{d~ zyS|^|g&w8wTxLn2m=PkrgM(N~{9zO239hd-;}`{XN)2PjV#$t2!Fn^B~rK@jFw&GZ=ev}4zwZl04U`yk9vD-t&zBu_!1 zKM>^_2tpOU$pnwqgOG<*GfdI!Fqlps0xhvN?9k_)BQZ<5OJN3M*i_a@0zo4zyB&h?p6Jb@-r_ws60h*7I-|Iq|D)@C?X7#4|~_g=vyo zoCVrc_dPE;M0xVV_=wWHHqkR`hJG)>Yvwy>9zI0gjEi9Y) zc1;u?4TIzOVG38tn#EYG=06>VD;5Jj*Ev9v8Q*S!`42lGeM#{johrqXbtpu97i|(0 z@e-ltccJT%{4ipxxaN6$qY`5P2$J4hazxGR#LF0*Q%5LWAV}#54M0KiJALN}B3>5y zg_7mR_V0~Y6JNbd5mG~De3Ut=dsb12V(hx--V(%LlrQlnkkh>()AFN~j2GKakK)7U zXsz0Rz|}?hwL_%s_^Z?x>WPDBL%E^6sy@nq-)tSEe`y^C@b`X2lM8O{Uqrj!dzb#G zbrY<76X3-AN9Y`X-$qLpm>?D=`7+u8g!6pry+v$#->1K7J1_fy{-S*o{~_JdzS%xR zcNe|!ekquS{N@ZHgceX>DSx`n}TDir0Y+`H(=DW5@Ay33380jm1j0=6PW zXvssV3Dfb~&mg(<@x=h3EH{30e6d3P@(O*aiO-Oik5cpz{%Kx9fPvp>C$1NEp32mS zJ1_mccIjqK45@B{>w=DmN!2qnIjzkDH+rSb-jQFKSj-t>y|3*IiFztypUWi9ao@GY zsszE_(?a--KHdXDs=dTWTWXb1$-!rQJyu3sm`rLeK^B2*T6Ur)`oI|b&Ua4`4^?O!3 zQyfiq9?YXA;=Xk9=E=+H+k9{a`LSaf`A_hf#0SL5j1?5%4GUXp9FBUjDU`ieP#ljm z(33GcpS1*x~IKDm>ex2g;y3(R3ySD+C|za^Te;MR6frI2$jeeYZBWEXa@Hc zP$;bK6wnmzE2L>`T7_KO!x7ioMw5A<1C=^-?#@L5PXj54 zEd>0qMbZ^g)AKZDo)&z^7E0+OGy+IU1#b_o^imAv` ziiPtxfYilhw9t7okUEGLmC$N-#c4Q~gI2PVM1CTUt%??MW+{bwz%q%e<5ALn#6ANG78UbOUVRJS)X@7lBc<$iSw0@^vl@qA;3Z zOBJJ+zzfRJ+kZ8p>Gy*nz|ai*94x0)!AOc#FmkP&ir5tb^D%8yR!{^V4cBsiqk@W@ z)=jjUIDI_?SycjTQUY9_$U`B+i5RV?jWne@1P@=WCp(u^X_hC`X=QgbBFcG^bIt+zWH=i4{Dm^AQ3{LQ_T!Tr%F z5QTg4Kn;5N&PF&ozDM(VXCqL`bd4Dg{!uM0P$b46fa|NtF8X3k5`8&dO+~Ux^37C( zQiNl!UyQbHwSbb?aA*dN_u;H%6w9sQsNcPMq@ag3BO5G$I5EZsXYm5j!~1n0462fd z;Cd?R#y8O+vtT?+hD;U-C=N8}AEQTd9C&^M;F~nl<*ms3Yy+ja?Wb_SHu@H?ZP32k zIzdlYBdB~sr1;-y^w=m~48x*E6b6KWZv;h>u_;1m~!j+chgW;P!1WzsA;S5Y%_Mj zvnEh*rx_Y~0U6ChZP;T&;S?pZ{r*lwz=66e1=%nS1Tf@cI_Y;&r2Y zBEJpLv3wr3ypYdhix_hYOpb<7m`AzbzM&OyuGVALk49q(9+is~)CQxHX3X90R#a!S zV;RZl5QD*(sNJ}uTdQ2CGnPl@Qv7yEUGKp`uYc)+yLwrSC9dEVcr!Nis zeeD{P(0S}wj7v(y;=B&Hh0hcz&DK>A!jTS26IY&rf3}R)aC$R&LEB%8C~rgDD7!FX zJ?-FnZ5hI$;Ra6F4WYWvA~?Jm`yp>1HpSSwJJf)6_ueeXY0Vy*EqAV6^Z(u+s0H~O zICKlh$!NyYS9_quyXAP2<09|B9eZguAKIt&?w!3L*!_}T3n7ek2ayB+Ck?qZ_+Nq{ zj1*G1x(^<9Hfrvw8(@;TA02-#R_=oGyyR_Uui(k?G)?X?Qzv-ASQxl11bnu4YE)`} zMoxCVIP(FoBf|AgEkZ3`1n0@~ggzVr;Joo05S8HoMUmpP9H6Tcd^c>ZtEj0g{cgpM zX6MKux)?eAg_AgCgM!y+@#9AE<1^xd{SZT!dX6senbQ=c&&KZx+VkWz-O>Y(V2?9& zcRb|VNQTA(iHyx`{uOn-S7I^Ot!4##LE_Q84B zy!!zSjvK-a*V*~4cW~`oyr9pqnE^M6+&|Hihnc^;jP+~sMeYwS{U0`e*W+DIq^>Hb zGUia1z9ZCY2YorAU*Et+sl)vS9{7mPv*DUvS!G@$UFDh8KmR09a~?gQ*Ey!smmLFo zyIJd^2YRSBGe$H}E%_}jp?)mD(i{48EA5Qx&?P@Ub3-4eqb5qGFJJ#iuVgUZkMz|6 zq*-m9XyB1Q>UEAi@OWJ380=Z1ddH7wB4&qZ?pmKV4_zr#ig5*MOhe z^+am)0!Q4|+cmiqUY`gpRo&LxHrNc0OOw2o+)KevQ}^0IedWk)5}c E1BaaxdjJ3c diff --git a/pkg/services/tree/service.proto b/pkg/services/tree/service.proto index 582e74847..88bf0bca4 100644 --- a/pkg/services/tree/service.proto +++ b/pkg/services/tree/service.proto @@ -242,8 +242,8 @@ message GetSubTreeRequest { bytes container_id = 1; // The name of the tree. string tree_id = 2; - // ID of the root node of a subtree. - uint64 root_id = 3; + // IDs of the root nodes of a subtree forest. + repeated uint64 root_id = 3 [ packed = false ]; // Optional depth of the traversal. Zero means return only root. // Maximum depth is 10. uint32 depth = 4; @@ -262,11 +262,11 @@ message GetSubTreeRequest { message GetSubTreeResponse { message Body { // ID of the node. - uint64 node_id = 1; + repeated uint64 node_id = 1 [ packed = false ]; // ID of the parent. - uint64 parent_id = 2; + repeated uint64 parent_id = 2 [ packed = false ]; // Time node was first added to a tree. - uint64 timestamp = 3; + repeated uint64 timestamp = 3 [ packed = false ]; // Node meta-information. repeated KeyValue meta = 4; } diff --git a/pkg/services/tree/service_frostfs.pb.go b/pkg/services/tree/service_frostfs.pb.go index 4b50af2aacb6f6140ecf03c8021c81ef169e7409..1a49c5c0c1d0cf110c458c168ddc145ff9bc71c8 100644 GIT binary patch delta 1399 zcmX?efqD5&<_Z208eE)7nR$stl?qC!dCB=HnR)5@5CJ7F&Vr)+lKk?_qErQ?^!)tv zoK(H^{G7zRbiMqdbbYX7QfZn#TpwJy5|`#ix5G7)k8}#g2Ic3Mc%~>+b8&JOXI7;u zXxl2l)#(KTMcj+>OA9m*Vt%E$NvTB|#+o{43d0hMGV@9_D)b;mMrX!qPF7?R*=$#< z#tgEm9?2?@bwKB*re#(rSRvc!3i4!XNMbr68zHuZR2HOag3QWKODj$-0l6L$I(h-6 zC81>y7Xbqy8YmwQRA3dWqfiQTrI`uPe30#$#Oj7>M)n=h-dZls+Q|od#5Qkgn87hw zHBU5(oUk;+9hQE;(C|zFg{9_Xg*>sOfW)HIJhCFp2)89LL*dq>gkJtlaJ|H8RPgcm4K=lbcw#fDhY8gSISE3+UnhdW%^E*7p!aafXTPy%1vIW$~{@iO_I@g^7|aE$-0tCn>~AcnI?%qv{eeoOiuHZ ZoBSY0cyfXUH;`>O`E!o%<~b9&qXDB7Fy;UN diff --git a/pkg/services/tree/types.pb.go b/pkg/services/tree/types.pb.go index 6464ccb776accfbcc4ad71c3630db3520dfae867..738b388614971ac85e6ab81790a8b264e75c9ceb 100644 GIT binary patch delta 92 zcmdn)``>3mH?xVJ(Z;?_vdoEjm7Bfg4m0rs89EA?3YmE&nhHSnWJe{j$H?y&x!N$H#vfP<@C8