From 3cc67db083d53ff9eb44a820ed2c87d4a26f011c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 23 May 2022 14:32:24 +0300 Subject: [PATCH] [#1419] pilorama: Create new nodes in path if needed Consider a node `{FileName: "dir", Attribute: "xxx"}`. In case we add a new node by path `["dir", "file.txt"]`, create a new intermediate node with a single attribute. `GetByPath` now also considers only nodes with a single attribute while building a path. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/boltdb.go | 12 ++--- pkg/local_object_storage/pilorama/forest.go | 5 +- .../pilorama/forest_test.go | 51 +++++++++++++++++-- pkg/local_object_storage/pilorama/inmemory.go | 5 +- .../pilorama/interface.go | 1 + 5 files changed, 58 insertions(+), 16 deletions(-) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 2cd389d44..4644786e1 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -470,14 +470,10 @@ loop: return 0, 0, err } - for j := range m.Items { - if m.Items[j].Key == attr { - if string(m.Items[j].Value) == path[i] { - curNode = child - continue loop - } - break - } + // Internal nodes have exactly one attribute. + if len(m.Items) == 1 && m.Items[0].Key == attr && string(m.Items[0].Value) == path[i] { + curNode = child + continue loop } childKey, _ = c.Next() } diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 1af95c922..255ca9ea3 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -59,9 +59,12 @@ func (f *memoryForest) TreeAddByPath(cid cidSDK.ID, treeID string, attr string, node = lm[j-i].Child s.operations = append(s.operations, lm[j-i]) } + + mCopy := make([]KeyValue, len(m)) + copy(mCopy, m) lm[len(lm)-1] = s.do(&Move{ Parent: node, - Meta: Meta{Time: s.timestamp(), Items: m}, + Meta: Meta{Time: s.timestamp(), Items: mCopy}, Child: s.findSpareID(), }) return lm, nil diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 585ca4252..4b80a5d4f 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -242,6 +242,44 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { require.Equal(t, 2, len(lm)) testMeta(t, s, cid, treeID, lm[0].Child, lm[0].Parent, Meta{Time: lm[0].Time, Items: []KeyValue{{AttributeFilename, []byte("dir")}}}) testMeta(t, s, cid, treeID, lm[1].Child, lm[1].Parent, Meta{Time: lm[1].Time, Items: meta}) + + t.Run("create internal nodes", func(t *testing.T) { + meta[0].Value = []byte("SomeValue") + meta[1].Value = []byte("another") + lm, err = s.TreeAddByPath(cid, treeID, AttributeFilename, []string{"path"}, meta) + require.NoError(t, err) + require.Equal(t, 1, len(lm)) + + oldMove := lm[0] + + meta[0].Value = []byte("Leaf") + meta[1].Value = []byte("file.txt") + lm, err = s.TreeAddByPath(cid, treeID, AttributeFilename, []string{"path", "another"}, meta) + require.NoError(t, err) + require.Equal(t, 2, len(lm)) + + testMeta(t, s, cid, treeID, lm[0].Child, lm[0].Parent, + Meta{Time: lm[0].Time, Items: []KeyValue{{AttributeFilename, []byte("another")}}}) + testMeta(t, s, cid, treeID, lm[1].Child, lm[1].Parent, Meta{Time: lm[1].Time, Items: meta}) + + require.NotEqual(t, lm[0].Child, oldMove.Child) + testMeta(t, s, cid, treeID, oldMove.Child, oldMove.Parent, + Meta{Time: oldMove.Time, Items: []KeyValue{ + {AttributeVersion, []byte("SomeValue")}, + {AttributeFilename, []byte("another")}}}) + + t.Run("get by path", func(t *testing.T) { + nodes, err := s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"path", "another"}, false) + require.NoError(t, err) + require.Equal(t, 2, len(nodes)) + require.ElementsMatch(t, []Node{lm[0].Child, oldMove.Child}, nodes) + + nodes, err = s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"path", "another", "file.txt"}, false) + require.NoError(t, err) + require.Equal(t, 1, len(nodes)) + require.Equal(t, lm[1].Child, nodes[0]) + }) + }) } func TestForest_Apply(t *testing.T) { @@ -475,15 +513,18 @@ func testTreeGetByPath(t *testing.T, s Forest) { } func testMove(t *testing.T, s Forest, ts int, node, parent Node, cid cidSDK.ID, treeID, filename, version string) { + items := make([]KeyValue, 1, 2) + items[0] = KeyValue{AttributeFilename, []byte(filename)} + if version != "" { + items = append(items, KeyValue{AttributeVersion, []byte(version)}) + } + require.NoError(t, s.TreeApply(cid, treeID, &Move{ Parent: parent, Child: node, Meta: Meta{ - Time: uint64(ts), - Items: []KeyValue{ - {AttributeFilename, []byte(filename)}, - {AttributeVersion, []byte(version)}, - }, + Time: uint64(ts), + Items: items, }, })) } diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index 91199cc1c..505cca26d 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -184,8 +184,9 @@ loop: for i := range path { children := t.childMap[curNode] for j := range children { - f := t.infoMap[children[j]].Meta.GetAttr(attr) - if string(f) == path[i] { + meta := t.infoMap[children[j]].Meta + f := meta.GetAttr(attr) + if len(meta.Items) == 1 && string(f) == path[i] { curNode = children[j] continue loop } diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index b0bf5f2bb..2a4054813 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -10,6 +10,7 @@ type Forest interface { TreeMove(cid cidSDK.ID, treeID string, m *Move) (*LogMove, error) // TreeAddByPath adds new node in the tree using provided path. // The path is constructed by descending from the root using the values of the attr in meta. + // Internal nodes in path should have exactly one attribute, otherwise a new node is created. TreeAddByPath(cid cidSDK.ID, treeID string, attr string, path []string, meta []KeyValue) ([]LogMove, error) // TreeApply applies replicated operation from another node. TreeApply(cid cidSDK.ID, treeID string, m *Move) error