TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 101 Pull requests 6 Projects Releases 1 Activity Actions

Simplify in-memory pilorama implementation #288

Merged
fyrchik merged 5 commits from fyrchik/frostfs-node:pilorama-simplify-inmemory into master 2023-04-28 07:56:21 +00:00
Conversation 0 Commits 5 Files changed 3 +46 -76
3 changed files with 46 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
pkg/local_object_storage/pilorama/forest.go
View file

@ -12,7 +12,7 @@ import (
// memoryForest represents multiple replicating trees sharing a single storage.
type memoryForest struct {
// treeMap maps tree identifier (container ID + name) to the replicated log.
treeMap map[string]*state
treeMap map[string]*memoryTree
}
var _ Forest = (*memoryForest)(nil)
@ -21,7 +21,7 @@ var _ Forest = (*memoryForest)(nil)
// TODO: this function will eventually be removed and is here for debugging.
func NewMemoryForest() ForestStorage {
return &memoryForest{
treeMap: make(map[string]*state),
treeMap: make(map[string]*memoryTree),
}
}
@ -34,7 +34,7 @@ func (f *memoryForest) TreeMove(_ context.Context, d CIDDescriptor, treeID strin
fullID := d.CID.String() + "/" + treeID
s, ok := f.treeMap[fullID]
if !ok {
s = newState()
s = newMemoryTree()
f.treeMap[fullID] = s
}
@ -60,7 +60,7 @@ func (f *memoryForest) TreeAddByPath(_ context.Context, d CIDDescriptor, treeID
fullID := d.CID.String() + "/" + treeID
s, ok := f.treeMap[fullID]
if !ok {
s = newState()
s = newMemoryTree()
f.treeMap[fullID] = s
}
@ -89,6 +89,7 @@ func (f *memoryForest) TreeAddByPath(_ context.Context, d CIDDescriptor, treeID
},
Child: s.findSpareID(),
})
s.operations = append(s.operations, op)
lm[len(lm)-1] = op.Move
return lm, nil
}
@ -98,7 +99,7 @@ func (f *memoryForest) TreeApply(_ context.Context, cnr cid.ID, treeID string, o
fullID := cnr.String() + "/" + treeID
s, ok := f.treeMap[fullID]
if !ok {
s = newState()
s = newMemoryTree()
f.treeMap[fullID] = s
}
@ -131,7 +132,7 @@ func (f *memoryForest) TreeGetByPath(_ context.Context, cid cid.ID, treeID strin
return nil, ErrTreeNotFound
}
return s.get(attr, path, latest), nil
return s.getByPath(attr, path, latest), nil
}
// TreeGetMeta implements the Forest interface.
@ -142,7 +143,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string,
return Meta{}, 0, ErrTreeNotFound
}
return s.getMeta(nodeID), s.infoMap[nodeID].Parent, nil
return s.infoMap[nodeID].Meta, s.infoMap[nodeID].Parent, nil
}
// TreeGetChildren implements the Forest interface.
@ -153,11 +154,7 @@ func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID str
return nil, ErrTreeNotFound
}
children, ok := s.childMap[nodeID]
if !ok {
return nil, nil
}
children := s.tree.getChildren(nodeID)
res := make([]Node, len(children))
copy(res, children)
return res, nil

11
pkg/local_object_storage/pilorama/forest_test.go
View file

@ -322,6 +322,10 @@ func testForestTreeAddByPath(t *testing.T, s Forest) {
firstID := lm[2].Child
testMeta(t, s, cid, treeID, firstID, lm[2].Parent, Meta{Time: lm[2].Time, Items: meta})
// TreeAddByPath must return operations in increasing time order.
require.True(t, lm[0].Time < lm[1].Time)
require.True(t, lm[1].Time < lm[2].Time)
meta[0].Value = []byte("YYY")
lm, err = s.TreeAddByPath(context.Background(), d, treeID, AttributeFilename, []string{"path", "to"}, meta)
require.NoError(t, err)
@ -714,13 +718,6 @@ func compareForests(t *testing.T, expected, actual Forest, cid cidSDK.ID, treeID
require.True(t, ok)
require.Equal(t, se.operations, sa.operations)
require.Equal(t, se.infoMap, sa.infoMap)
require.Equal(t, len(se.childMap), len(sa.childMap))
for ck, la := range sa.childMap {
le, ok := se.childMap[ck]
require.True(t, ok)
require.ElementsMatch(t, le, la)
}
}
require.Equal(t, expected, actual, i)
}

90
pkg/local_object_storage/pilorama/inmemory.go
View file

@ -1,5 +1,7 @@
package pilorama
import "sort"
// nodeInfo couples parent and metadata.
type nodeInfo struct {
Parent Node
@ -12,42 +14,25 @@ type move struct {
Old nodeInfo
}
// state represents state being replicated.
type state struct {
// memoryTree represents memoryTree being replicated.
type memoryTree struct {
operations []move
tree
}
// newState constructs new empty tree.
func newState() *state {
return &state{
tree: *newTree(),
// newMemoryTree constructs new empty tree.
func newMemoryTree() *memoryTree {
return &memoryTree{
tree: tree{
infoMap: make(map[Node]nodeInfo),
},
}
}
// undo un-does op and changes s in-place.
func (s *state) undo(op *move) {
children := s.tree.childMap[op.Parent]
for i := range children {
if children[i] == op.Child {
if len(children) > 1 {
s.tree.childMap[op.Parent] = append(children[:i], children[i+1:]...)
} else {
delete(s.tree.childMap, op.Parent)
}
break
}
}
func (s *memoryTree) undo(op *move) {
if op.HasOld {
s.tree.infoMap[op.Child] = op.Old
oldChildren := s.tree.childMap[op.Old.Parent]
for i := range oldChildren {
if oldChildren[i] == op.Child {
return
}
}
s.tree.childMap[op.Old.Parent] = append(oldChildren, op.Child)
} else {
delete(s.tree.infoMap, op.Child)
}
@ -55,7 +40,7 @@ func (s *state) undo(op *move) {
// Apply puts op in log at a proper position, re-applies all subsequent operations
// from log and changes s in-place.
func (s *state) Apply(op *Move) error {
func (s *memoryTree) Apply(op *Move) error {
var index int
for index = len(s.operations); index > 0; index-- {
if s.operations[index-1].Time <= op.Time {
@ -82,7 +67,7 @@ func (s *state) Apply(op *Move) error {
}
// do performs a single move operation on a tree.
func (s *state) do(op *Move) move {
func (s *memoryTree) do(op *Move) move {
lm := move{
Move: Move{
Parent: op.Parent,
@ -104,36 +89,23 @@ func (s *state) do(op *Move) move {
if !ok {
p.Meta.Time = op.Time
} else {
s.removeChild(op.Child, p.Parent)
}
p.Meta = op.Meta
p.Parent = op.Parent
s.tree.infoMap[op.Child] = p
s.tree.childMap[op.Parent] = append(s.tree.childMap[op.Parent], op.Child)
return lm
}
func (s *state) removeChild(child, parent Node) {
oldChildren := s.tree.childMap[parent]
for i := range oldChildren {
if oldChildren[i] == child {
s.tree.childMap[parent] = append(oldChildren[:i], oldChildren[i+1:]...)
break
}
}
}
func (s *state) timestamp(pos, size int) Timestamp {
func (s *memoryTree) timestamp(pos, size int) Timestamp {
if len(s.operations) == 0 {
return nextTimestamp(0, uint64(pos), uint64(size))
}
return nextTimestamp(s.operations[len(s.operations)-1].Time, uint64(pos), uint64(size))
}
func (s *state) findSpareID() Node {
func (s *memoryTree) findSpareID() Node {
id := uint64(1)
for _, ok := s.infoMap[id]; ok; _, ok = s.infoMap[id] {
id++
@ -145,14 +117,22 @@ func (s *state) findSpareID() Node {
type tree struct {
syncHeight uint64
infoMap map[Node]nodeInfo
childMap map[Node][]Node
}
func newTree() *tree {
return &tree{
childMap: make(map[Node][]Node),
infoMap: make(map[Node]nodeInfo),
func (t tree) getChildren(parent Node) []Node {
var children []Node
for c, info := range t.infoMap {
if info.Parent == parent {
children = append(children, c)
}
}
sort.Slice(children, func(i, j int) bool {
a := t.infoMap[children[i]]
b := t.infoMap[children[j]]
return a.Meta.Time < b.Meta.Time
})
return children
}
// isAncestor returns true if parent is an ancestor of a child.
@ -176,7 +156,7 @@ func (t tree) getPathPrefix(attr string, path []string) (int, Node) {
loop:
for i := range path {
children := t.childMap[curNode]
children := t.getChildren(curNode)
for j := range children {
meta := t.infoMap[children[j]].Meta
f := meta.GetAttr(attr)
@ -191,9 +171,10 @@ loop:
return len(path), curNode
}
// get returns list of nodes which have the specified path from root
// getByPath returns list of nodes which have the specified path from root
// descending by values of attr from meta.
func (t tree) get(attr string, path []string, latest bool) []Node {
// If latest is true, only the latest node is returned.
func (t tree) getByPath(attr string, path []string, latest bool) []Node {
if len(path) == 0 {
return nil
}
@ -206,7 +187,7 @@ func (t tree) get(attr string, path []string, latest bool) []Node {
var nodes []Node
var lastTs Timestamp
children := t.childMap[curNode]
children := t.getChildren(curNode)
for i := range children {
info := t.infoMap[children[i]]
fileName := string(info.Meta.GetAttr(attr))
@ -223,8 +204,3 @@ func (t tree) get(attr string, path []string, latest bool) []Node {
return nodes
}
// getMeta returns meta information of node n.
func (t tree) getMeta(n Node) Meta {
return t.infoMap[n].Meta
}