[#957] treesvc: Abstract out Heap interface for listing

```
goos: linux
goarch: amd64
pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree
cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz
                        │     old      │                 new                  │
                        │    sec/op    │    sec/op     vs base                │
GetSubTree/latency-8      3.542m ± 12%   4.059m ± 20%  +14.59% (p=0.005 n=10)
GetSubTree/total_time-8   73.19m ±  9%   78.82m ±  4%   +7.69% (p=0.043 n=10)
geomean                   16.10m         17.89m        +11.09%

                        │     old      │                 new                  │
                        │     B/op     │     B/op      vs base                │
GetSubTree/latency-8      28.23Mi ± 0%   32.81Mi ± 0%  +16.22% (p=0.000 n=10)
GetSubTree/total_time-8   28.23Mi ± 0%   32.81Mi ± 0%  +16.22% (p=0.000 n=10)
geomean                   28.23Mi        32.81Mi       +16.22%

                        │     old     │                new                 │
                        │  allocs/op  │  allocs/op   vs base               │
GetSubTree/latency-8      400.0k ± 0%   400.0k ± 0%  +0.00% (p=0.000 n=10)
GetSubTree/total_time-8   400.0k ± 0%   400.0k ± 0%  +0.00% (p=0.000 n=10)
geomean                   400.0k        400.0k       +0.00%
```

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
Evgenii Stratonikov 2024-02-02 23:26:47 +03:00
parent 7eab9fcede
commit 3601feab81
3 changed files with 104 additions and 22 deletions

View file

@ -0,0 +1,47 @@
package heap
import (
"bytes"
"sort"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
)
type OrderedSlice struct {
nodes []pilorama.NodeInfo
sorted bool
}
func NewOrderedSlice() *OrderedSlice {
return new(OrderedSlice)
}
func (s *OrderedSlice) Insert(infos ...pilorama.NodeInfo) {
s.sorted = false
s.nodes = append(s.nodes, infos...)
}
func (s *OrderedSlice) IsEmpty() bool {
return len(s.nodes) == 0
}
func (s *OrderedSlice) ExtractMin() pilorama.NodeInfo {
if !s.sorted {
sortByFilename(s.nodes)
s.sorted = true
}
node := s.nodes[0]
s.nodes = s.nodes[1:]
return node
}
func sortByFilename(nodes []pilorama.NodeInfo) {
if len(nodes) == 0 {
return
}
less := func(i, j int) bool {
return bytes.Compare(nodes[i].Meta.GetAttr(pilorama.AttributeFilename), nodes[j].Meta.GetAttr(pilorama.AttributeFilename)) < 0
}
sort.Slice(nodes, less)
}

View file

@ -0,0 +1,25 @@
package heap
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
)
type UnorderedSlice []pilorama.NodeInfo
func NewUnorderedSlice() *UnorderedSlice {
return &UnorderedSlice{}
}
func (s *UnorderedSlice) Insert(infos ...pilorama.NodeInfo) {
*s = append(*s, infos...)
}
func (s *UnorderedSlice) IsEmpty() bool {
return len(*s) == 0
}
func (s *UnorderedSlice) ExtractMin() pilorama.NodeInfo {
node := (*s)[0]
*s = (*s)[1:]
return node
}

View file

@ -5,11 +5,11 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"sort"
"sync" "sync"
"sync/atomic" "sync/atomic"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree/heap"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -440,29 +440,48 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS
return getSubTree(srv.Context(), srv, cid, b, s.forest) return getSubTree(srv.Context(), srv, cid, b, s.forest)
} }
type Heap interface {
Insert(...pilorama.NodeInfo)
IsEmpty() bool
ExtractMin() pilorama.NodeInfo
}
func makeHeap(ordered bool) Heap {
if ordered {
return heap.NewOrderedSlice()
}
return heap.NewUnorderedSlice()
}
func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error {
ordered, err := needOrder(b.GetOrderBy().GetDirection())
if err != nil {
return err
}
// Traverse the tree in a DFS manner. Because we need to support arbitrary depth, // Traverse the tree in a DFS manner. Because we need to support arbitrary depth,
// recursive implementation is not suitable here, so we maintain explicit stack. // recursive implementation is not suitable here, so we maintain explicit stack.
m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId()) m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId())
if err != nil { if err != nil {
return err return err
} }
stack := [][]pilorama.NodeInfo{{{
stack := []Heap{makeHeap(ordered)}
stack[0].Insert(pilorama.NodeInfo{
ID: b.GetRootId(), ID: b.GetRootId(),
Meta: m, Meta: m,
ParentID: p, ParentID: p,
}}} })
for { for {
if len(stack) == 0 { if len(stack) == 0 {
break break
} else if len(stack[len(stack)-1]) == 0 { } else if stack[len(stack)-1].IsEmpty() {
stack = stack[:len(stack)-1] stack = stack[:len(stack)-1]
continue continue
} }
node := stack[len(stack)-1][0] node := stack[len(stack)-1].ExtractMin()
stack[len(stack)-1] = stack[len(stack)-1][1:]
err = srv.Send(&GetSubTreeResponse{ err = srv.Send(&GetSubTreeResponse{
Body: &GetSubTreeResponse_Body{ Body: &GetSubTreeResponse_Body{
@ -481,33 +500,24 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD
if err != nil { if err != nil {
return err return err
} }
children, err = sortByFilename(children, b.GetOrderBy().GetDirection())
if err != nil {
return err
}
if len(children) != 0 { if len(children) != 0 {
stack = append(stack, children) h := makeHeap(ordered)
h.Insert(children...)
stack = append(stack, h)
} }
} }
} }
return nil return nil
} }
func sortByFilename(nodes []pilorama.NodeInfo, d GetSubTreeRequest_Body_Order_Direction) ([]pilorama.NodeInfo, error) { func needOrder(d GetSubTreeRequest_Body_Order_Direction) (bool, error) {
switch d { switch d {
case GetSubTreeRequest_Body_Order_None: case GetSubTreeRequest_Body_Order_None:
return nodes, nil return false, nil
case GetSubTreeRequest_Body_Order_Asc: case GetSubTreeRequest_Body_Order_Asc:
if len(nodes) == 0 { return true, nil
return nodes, nil
}
less := func(i, j int) bool {
return bytes.Compare(nodes[i].Meta.GetAttr(pilorama.AttributeFilename), nodes[j].Meta.GetAttr(pilorama.AttributeFilename)) < 0
}
sort.Slice(nodes, less)
return nodes, nil
default: default:
return nil, fmt.Errorf("unsupported order direction: %s", d.String()) return false, fmt.Errorf("unsupported order direction: %s", d.String())
} }
} }