From 43ee671f364465d003e2fbf6d9cefd3a0f9afca2 Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Tue, 3 Aug 2021 14:19:50 +0300 Subject: [PATCH 1/5] mpt: do not allocate NodeObject for serialization Signed-off-by: Evgeniy Stratonikov --- pkg/core/mpt/base.go | 10 +++++++++- pkg/core/mpt/branch.go | 8 +------- pkg/core/mpt/extension.go | 8 +------- pkg/core/mpt/hash.go | 6 ------ pkg/core/mpt/leaf.go | 6 ------ 5 files changed, 11 insertions(+), 27 deletions(-) diff --git a/pkg/core/mpt/base.go b/pkg/core/mpt/base.go index 8762281d6..63e56dc52 100644 --- a/pkg/core/mpt/base.go +++ b/pkg/core/mpt/base.go @@ -23,7 +23,6 @@ type BaseNodeIface interface { Hash() util.Uint256 Type() NodeType Bytes() []byte - EncodeBinaryAsChild(w *io.BinWriter) } type flushedNode interface { @@ -76,6 +75,15 @@ func (b *BaseNode) invalidateCache() { b.hashValid = false } +func encodeBinaryAsChild(n Node, w *io.BinWriter) { + if isEmpty(n) { + w.WriteB(byte(EmptyT)) + return + } + w.WriteB(byte(HashT)) + w.WriteBytes(n.Hash().BytesBE()) +} + // encodeNodeWithType encodes node together with it's type. func encodeNodeWithType(n Node, w *io.BinWriter) { switch t := n.Type(); t { diff --git a/pkg/core/mpt/branch.go b/pkg/core/mpt/branch.go index e01c02620..d66679d8f 100644 --- a/pkg/core/mpt/branch.go +++ b/pkg/core/mpt/branch.go @@ -48,16 +48,10 @@ func (b *BranchNode) Bytes() []byte { // EncodeBinary implements io.Serializable. func (b *BranchNode) EncodeBinary(w *io.BinWriter) { for i := 0; i < childrenCount; i++ { - b.Children[i].EncodeBinaryAsChild(w) + encodeBinaryAsChild(b.Children[i], w) } } -// EncodeBinaryAsChild implements BaseNode interface. -func (b *BranchNode) EncodeBinaryAsChild(w *io.BinWriter) { - n := &NodeObject{Node: NewHashNode(b.Hash())} // with type - n.EncodeBinary(w) -} - // DecodeBinary implements io.Serializable. func (b *BranchNode) DecodeBinary(r *io.BinReader) { for i := 0; i < childrenCount; i++ { diff --git a/pkg/core/mpt/extension.go b/pkg/core/mpt/extension.go index 7cb0bb7f0..1b8047e20 100644 --- a/pkg/core/mpt/extension.go +++ b/pkg/core/mpt/extension.go @@ -69,13 +69,7 @@ func (e *ExtensionNode) DecodeBinary(r *io.BinReader) { // EncodeBinary implements io.Serializable. func (e ExtensionNode) EncodeBinary(w *io.BinWriter) { w.WriteVarBytes(e.key) - e.next.EncodeBinaryAsChild(w) -} - -// EncodeBinaryAsChild implements BaseNode interface. -func (e *ExtensionNode) EncodeBinaryAsChild(w *io.BinWriter) { - n := &NodeObject{Node: NewHashNode(e.Hash())} // with type - n.EncodeBinary(w) + encodeBinaryAsChild(e.next, w) } // MarshalJSON implements json.Marshaler. diff --git a/pkg/core/mpt/hash.go b/pkg/core/mpt/hash.go index ca24dc457..f34b542f2 100644 --- a/pkg/core/mpt/hash.go +++ b/pkg/core/mpt/hash.go @@ -58,12 +58,6 @@ func (h HashNode) EncodeBinary(w *io.BinWriter) { w.WriteBytes(h.hash[:]) } -// EncodeBinaryAsChild implements BaseNode interface. -func (h *HashNode) EncodeBinaryAsChild(w *io.BinWriter) { - no := &NodeObject{Node: h} // with type - no.EncodeBinary(w) -} - // MarshalJSON implements json.Marshaler. func (h *HashNode) MarshalJSON() ([]byte, error) { if !h.hashValid { diff --git a/pkg/core/mpt/leaf.go b/pkg/core/mpt/leaf.go index 49ee55f97..ecb003c23 100644 --- a/pkg/core/mpt/leaf.go +++ b/pkg/core/mpt/leaf.go @@ -56,12 +56,6 @@ func (n LeafNode) EncodeBinary(w *io.BinWriter) { w.WriteVarBytes(n.value) } -// EncodeBinaryAsChild implements BaseNode interface. -func (n *LeafNode) EncodeBinaryAsChild(w *io.BinWriter) { - no := &NodeObject{Node: NewHashNode(n.Hash())} // with type - no.EncodeBinary(w) -} - // MarshalJSON implements json.Marshaler. func (n *LeafNode) MarshalJSON() ([]byte, error) { return []byte(`{"value":"` + hex.EncodeToString(n.value) + `"}`), nil From 291a29af1e1f70bbe16574b3d53dafaee19bc82f Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Fri, 6 Aug 2021 11:55:43 +0300 Subject: [PATCH 2/5] *: do not use `WriteArray` for frequently used items MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `WriteArray` involves reflection, it makes sense to optimize serialization of transactions and application logs which are serialized constantly. Adding case in a type switch in `WriteArray` is not an option because we don't want new dependencies for `io` package. ``` name old time/op new time/op delta AppExecResult_EncodeBinary-8 852ns ± 3% 656ns ± 2% -22.94% (p=0.000 n=10+9) name old alloc/op new alloc/op delta AppExecResult_EncodeBinary-8 448B ± 0% 376B ± 0% -16.07% (p=0.000 n=10+10) name old allocs/op new allocs/op delta AppExecResult_EncodeBinary-8 7.00 ± 0% 5.00 ± 0% -28.57% (p=0.000 n=10+10) ``` ``` name old time/op new time/op delta Transaction_Bytes-8 1.29µs ± 3% 0.76µs ± 5% -41.52% (p=0.000 n=9+10) name old alloc/op new alloc/op delta Transaction_Bytes-8 1.21kB ± 0% 1.01kB ± 0% -16.56% (p=0.000 n=10+10) name old allocs/op new allocs/op delta Transaction_Bytes-8 12.0 ± 0% 7.0 ± 0% -41.67% (p=0.000 n=10+10) ``` Signed-off-by: Evgeniy Stratonikov --- pkg/core/state/notification_event.go | 5 ++++- pkg/core/state/notification_event_test.go | 25 +++++++++++++++++++++++ pkg/core/transaction/bench_test.go | 11 ++++++++++ pkg/core/transaction/transaction.go | 15 +++++++++++--- 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/pkg/core/state/notification_event.go b/pkg/core/state/notification_event.go index 5c7962ca0..c40f63e35 100644 --- a/pkg/core/state/notification_event.go +++ b/pkg/core/state/notification_event.go @@ -61,7 +61,10 @@ func (aer *AppExecResult) EncodeBinary(w *io.BinWriter) { for _, it := range aer.Stack { stackitem.EncodeBinaryProtected(it, w) } - w.WriteArray(aer.Events) + w.WriteVarUint(uint64(len(aer.Events))) + for i := range aer.Events { + aer.Events[i].EncodeBinary(w) + } w.WriteVarBytes([]byte(aer.FaultException)) } diff --git a/pkg/core/state/notification_event_test.go b/pkg/core/state/notification_event_test.go index 2f6667f61..896e2b82a 100644 --- a/pkg/core/state/notification_event_test.go +++ b/pkg/core/state/notification_event_test.go @@ -13,6 +13,31 @@ import ( "github.com/stretchr/testify/require" ) +func BenchmarkAppExecResult_EncodeBinary(b *testing.B) { + aer := &AppExecResult{ + Container: random.Uint256(), + Execution: Execution{ + Trigger: trigger.Application, + VMState: vm.HaltState, + GasConsumed: 12345, + Stack: []stackitem.Item{}, + Events: []NotificationEvent{{ + ScriptHash: random.Uint160(), + Name: "Event", + Item: stackitem.NewArray([]stackitem.Item{stackitem.NewBool(true)}), + }}, + }, + } + + w := io.NewBufBinWriter() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + w.Reset() + aer.EncodeBinary(w.BinWriter) + } +} + func TestEncodeDecodeNotificationEvent(t *testing.T) { event := &NotificationEvent{ ScriptHash: random.Uint160(), diff --git a/pkg/core/transaction/bench_test.go b/pkg/core/transaction/bench_test.go index 1cf85a1cb..991d729dc 100644 --- a/pkg/core/transaction/bench_test.go +++ b/pkg/core/transaction/bench_test.go @@ -53,3 +53,14 @@ func BenchmarkDecodeFromBytes(t *testing.B) { require.NoError(t, err) } } + +func BenchmarkTransaction_Bytes(b *testing.B) { + tx, err := NewTransactionFromBytes(benchTx) + require.NoError(b, err) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = tx.Bytes() + } +} diff --git a/pkg/core/transaction/transaction.go b/pkg/core/transaction/transaction.go index 7dd9b8ccf..012ceb142 100644 --- a/pkg/core/transaction/transaction.go +++ b/pkg/core/transaction/transaction.go @@ -203,7 +203,10 @@ func (t *Transaction) DecodeBinary(br *io.BinReader) { // EncodeBinary implements Serializable interface. func (t *Transaction) EncodeBinary(bw *io.BinWriter) { t.encodeHashableFields(bw) - bw.WriteArray(t.Scripts) + bw.WriteVarUint(uint64(len(t.Scripts))) + for i := range t.Scripts { + t.Scripts[i].EncodeBinary(bw) + } } // encodeHashableFields encodes the fields that are not used for @@ -218,8 +221,14 @@ func (t *Transaction) encodeHashableFields(bw *io.BinWriter) { bw.WriteU64LE(uint64(t.SystemFee)) bw.WriteU64LE(uint64(t.NetworkFee)) bw.WriteU32LE(t.ValidUntilBlock) - bw.WriteArray(t.Signers) - bw.WriteArray(t.Attributes) + bw.WriteVarUint(uint64(len(t.Signers))) + for i := range t.Signers { + t.Signers[i].EncodeBinary(bw) + } + bw.WriteVarUint(uint64(len(t.Attributes))) + for i := range t.Attributes { + t.Attributes[i].EncodeBinary(bw) + } bw.WriteVarBytes(t.Script) } From f02d8b4ec4c3fd38e31994c2c1af8957e145c058 Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Tue, 3 Aug 2021 16:29:21 +0300 Subject: [PATCH 3/5] stackitem: serialize integers to the pre-allocated slice Signed-off-by: Evgeniy Stratonikov --- pkg/vm/stackitem/serialization.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/vm/stackitem/serialization.go b/pkg/vm/stackitem/serialization.go index b9a814e46..84e01bedc 100644 --- a/pkg/vm/stackitem/serialization.go +++ b/pkg/vm/stackitem/serialization.go @@ -112,8 +112,11 @@ func (w *serContext) serialize(item Item) error { } case *BigInteger: w.data = append(w.data, byte(IntegerT)) - data := bigint.ToBytes(t.Value().(*big.Int)) - w.appendVarUint(uint64(len(data))) + v := t.Value().(*big.Int) + ln := len(w.data) + w.data = append(w.data, 0) + data := bigint.ToPreallocatedBytes(v, w.data[len(w.data):]) + w.data[ln] = byte(len(data)) w.data = append(w.data, data...) case *Interop: if w.allowInvalid { From db80ef28df1a904b2fe8c8fd65102c1ea353eee7 Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Tue, 3 Aug 2021 17:10:46 +0300 Subject: [PATCH 4/5] mpt: move empty hash node in a separate type We use them quite frequently (consider children for a new branch node) and it is better to get rid of unneeded allocations. Signed-off-by: Evgeniy Stratonikov --- pkg/core/mpt/base.go | 22 +++------------- pkg/core/mpt/batch.go | 27 +++++++++---------- pkg/core/mpt/batch_test.go | 41 ++++++++++++++--------------- pkg/core/mpt/branch.go | 2 +- pkg/core/mpt/empty.go | 53 ++++++++++++++++++++++++++++++++++++++ pkg/core/mpt/hash.go | 6 ----- pkg/core/mpt/node.go | 2 +- pkg/core/mpt/node_test.go | 5 +--- pkg/core/mpt/proof.go | 10 +++---- pkg/core/mpt/trie.go | 43 +++++++++++++++---------------- pkg/core/mpt/trie_test.go | 12 ++++----- 11 files changed, 124 insertions(+), 99 deletions(-) create mode 100644 pkg/core/mpt/empty.go diff --git a/pkg/core/mpt/base.go b/pkg/core/mpt/base.go index 63e56dc52..f1cd5864d 100644 --- a/pkg/core/mpt/base.go +++ b/pkg/core/mpt/base.go @@ -54,8 +54,8 @@ func (b *BaseNode) getBytes(n Node) []byte { // updateHash updates hash field for this BaseNode. func (b *BaseNode) updateHash(n Node) { - if n.Type() == HashT { - panic("can't update hash for hash node") + if n.Type() == HashT || n.Type() == EmptyT { + panic("can't update hash for empty or hash node") } b.hash = hash.DoubleSha256(b.getBytes(n)) b.hashValid = true @@ -86,17 +86,7 @@ func encodeBinaryAsChild(n Node, w *io.BinWriter) { // encodeNodeWithType encodes node together with it's type. func encodeNodeWithType(n Node, w *io.BinWriter) { - switch t := n.Type(); t { - case HashT: - hn := n.(*HashNode) - if !hn.hashValid { - w.WriteB(byte(EmptyT)) - break - } - fallthrough - default: - w.WriteB(byte(t)) - } + w.WriteB(byte(n.Type())) n.EncodeBinary(w) } @@ -120,11 +110,7 @@ func DecodeNodeWithType(r *io.BinReader) Node { case LeafT: n = new(LeafNode) case EmptyT: - n = &HashNode{ - BaseNode: BaseNode{ - hashValid: false, - }, - } + n = EmptyNode{} default: r.Err = fmt.Errorf("invalid node type: %x", typ) return nil diff --git a/pkg/core/mpt/batch.go b/pkg/core/mpt/batch.go index a03a6cbc8..6a6aac7a8 100644 --- a/pkg/core/mpt/batch.go +++ b/pkg/core/mpt/batch.go @@ -62,6 +62,8 @@ func (t *Trie) putBatchIntoNode(curr Node, kv []keyValue) (Node, int, error) { return t.putBatchIntoExtension(n, kv) case *HashNode: return t.putBatchIntoHash(n, kv) + case EmptyNode: + return t.putBatchIntoEmpty(kv) default: panic("invalid MPT node type") } @@ -84,11 +86,9 @@ func (t *Trie) mergeExtension(prefix []byte, sub Node) (Node, error) { sn.invalidateCache() t.addRef(sn.Hash(), sn.bytes) return sn, nil + case EmptyNode: + return sn, nil case *HashNode: - if sn.IsEmpty() { - return sn, nil - } - n, err := t.getFromStore(sn.Hash()) if err != nil { return sn, err @@ -141,8 +141,8 @@ func (t *Trie) putBatchIntoExtensionNoPrefix(key []byte, next Node, kv []keyValu } func isEmpty(n Node) bool { - hn, ok := n.(*HashNode) - return ok && hn.IsEmpty() + _, ok := n.(EmptyNode) + return ok } // addToBranch puts items into the branch node assuming b is not yet in trie. @@ -190,7 +190,7 @@ func (t *Trie) stripBranch(b *BranchNode) (Node, error) { } switch { case n == 0: - return new(HashNode), nil + return EmptyNode{}, nil case n == 1: if lastIndex != lastChild { return t.mergeExtension([]byte{lastIndex}, b.Children[lastIndex]) @@ -219,12 +219,13 @@ func (t *Trie) iterateBatch(kv []keyValue, f func(c byte, kv []keyValue) (int, e return n, nil } +func (t *Trie) putBatchIntoEmpty(kv []keyValue) (Node, int, error) { + common := lcpMany(kv) + stripPrefix(len(common), kv) + return t.newSubTrieMany(common, kv, nil) +} + func (t *Trie) putBatchIntoHash(curr *HashNode, kv []keyValue) (Node, int, error) { - if curr.IsEmpty() { - common := lcpMany(kv) - stripPrefix(len(common), kv) - return t.newSubTrieMany(common, kv, nil) - } result, err := t.getFromStore(curr.hash) if err != nil { return curr, 0, err @@ -242,7 +243,7 @@ func (t *Trie) newSubTrieMany(prefix []byte, kv []keyValue, value []byte) (Node, if len(kv[0].key) == 0 { if len(kv[0].value) == 0 { if len(kv) == 1 { - return new(HashNode), 1, nil + return EmptyNode{}, 1, nil } node, n, err := t.newSubTrieMany(prefix, kv[1:], nil) return node, n + 1, err diff --git a/pkg/core/mpt/batch_test.go b/pkg/core/mpt/batch_test.go index 6585d5204..47bfc4a6f 100644 --- a/pkg/core/mpt/batch_test.go +++ b/pkg/core/mpt/batch_test.go @@ -68,8 +68,8 @@ func testPut(t *testing.T, ps pairs, tr1, tr2 *Trie) { func TestTrie_PutBatchLeaf(t *testing.T) { prepareLeaf := func(t *testing.T) (*Trie, *Trie) { - tr1 := NewTrie(new(HashNode), false, newTestStore()) - tr2 := NewTrie(new(HashNode), false, newTestStore()) + tr1 := NewTrie(EmptyNode{}, false, newTestStore()) + tr2 := NewTrie(EmptyNode{}, false, newTestStore()) require.NoError(t, tr1.Put([]byte{0}, []byte("value"))) require.NoError(t, tr2.Put([]byte{0}, []byte("value"))) return tr1, tr2 @@ -97,8 +97,8 @@ func TestTrie_PutBatchLeaf(t *testing.T) { func TestTrie_PutBatchExtension(t *testing.T) { prepareExtension := func(t *testing.T) (*Trie, *Trie) { - tr1 := NewTrie(new(HashNode), false, newTestStore()) - tr2 := NewTrie(new(HashNode), false, newTestStore()) + tr1 := NewTrie(EmptyNode{}, false, newTestStore()) + tr2 := NewTrie(EmptyNode{}, false, newTestStore()) require.NoError(t, tr1.Put([]byte{1, 2}, []byte("value1"))) require.NoError(t, tr2.Put([]byte{1, 2}, []byte("value1"))) return tr1, tr2 @@ -144,8 +144,8 @@ func TestTrie_PutBatchExtension(t *testing.T) { func TestTrie_PutBatchBranch(t *testing.T) { prepareBranch := func(t *testing.T) (*Trie, *Trie) { - tr1 := NewTrie(new(HashNode), false, newTestStore()) - tr2 := NewTrie(new(HashNode), false, newTestStore()) + tr1 := NewTrie(EmptyNode{}, false, newTestStore()) + tr2 := NewTrie(EmptyNode{}, false, newTestStore()) require.NoError(t, tr1.Put([]byte{0x00, 2}, []byte("value1"))) require.NoError(t, tr2.Put([]byte{0x00, 2}, []byte("value1"))) require.NoError(t, tr1.Put([]byte{0x10, 3}, []byte("value2"))) @@ -175,8 +175,8 @@ func TestTrie_PutBatchBranch(t *testing.T) { require.IsType(t, (*ExtensionNode)(nil), tr1.root) }) t.Run("non-empty child is last node", func(t *testing.T) { - tr1 := NewTrie(new(HashNode), false, newTestStore()) - tr2 := NewTrie(new(HashNode), false, newTestStore()) + tr1 := NewTrie(EmptyNode{}, false, newTestStore()) + tr2 := NewTrie(EmptyNode{}, false, newTestStore()) require.NoError(t, tr1.Put([]byte{0x00, 2}, []byte("value1"))) require.NoError(t, tr2.Put([]byte{0x00, 2}, []byte("value1"))) require.NoError(t, tr1.Put([]byte{0x00}, []byte("value2"))) @@ -222,8 +222,8 @@ func TestTrie_PutBatchBranch(t *testing.T) { func TestTrie_PutBatchHash(t *testing.T) { prepareHash := func(t *testing.T) (*Trie, *Trie) { - tr1 := NewTrie(new(HashNode), false, newTestStore()) - tr2 := NewTrie(new(HashNode), false, newTestStore()) + tr1 := NewTrie(EmptyNode{}, false, newTestStore()) + tr2 := NewTrie(EmptyNode{}, false, newTestStore()) require.NoError(t, tr1.Put([]byte{0x10}, []byte("value1"))) require.NoError(t, tr2.Put([]byte{0x10}, []byte("value1"))) require.NoError(t, tr1.Put([]byte{0x20}, []byte("value2"))) @@ -257,8 +257,8 @@ func TestTrie_PutBatchHash(t *testing.T) { func TestTrie_PutBatchEmpty(t *testing.T) { t.Run("good", func(t *testing.T) { - tr1 := NewTrie(new(HashNode), false, newTestStore()) - tr2 := NewTrie(new(HashNode), false, newTestStore()) + tr1 := NewTrie(EmptyNode{}, false, newTestStore()) + tr2 := NewTrie(EmptyNode{}, false, newTestStore()) var ps = pairs{ {[]byte{0}, []byte("value0")}, {[]byte{1}, []byte("value1")}, @@ -273,15 +273,15 @@ func TestTrie_PutBatchEmpty(t *testing.T) { {[]byte{2}, nil}, {[]byte{3}, []byte("replace3")}, } - tr1 := NewTrie(new(HashNode), false, newTestStore()) - tr2 := NewTrie(new(HashNode), false, newTestStore()) + tr1 := NewTrie(EmptyNode{}, false, newTestStore()) + tr2 := NewTrie(EmptyNode{}, false, newTestStore()) testIncompletePut(t, ps, 4, tr1, tr2) }) } // For the sake of coverage. func TestTrie_InvalidNodeType(t *testing.T) { - tr := NewTrie(new(HashNode), false, newTestStore()) + tr := NewTrie(EmptyNode{}, false, newTestStore()) var b Batch b.Add([]byte{1}, []byte("value")) tr.root = Node(nil) @@ -289,8 +289,8 @@ func TestTrie_InvalidNodeType(t *testing.T) { } func TestTrie_PutBatch(t *testing.T) { - tr1 := NewTrie(new(HashNode), false, newTestStore()) - tr2 := NewTrie(new(HashNode), false, newTestStore()) + tr1 := NewTrie(EmptyNode{}, false, newTestStore()) + tr2 := NewTrie(EmptyNode{}, false, newTestStore()) var ps = pairs{ {[]byte{1}, []byte{1}}, {[]byte{2}, []byte{3}}, @@ -312,11 +312,10 @@ var _ = printNode // `spew.Dump()`. func printNode(prefix string, n Node) { switch tn := n.(type) { + case EmptyNode: + fmt.Printf("%s empty\n", prefix) + return case *HashNode: - if tn.IsEmpty() { - fmt.Printf("%s empty\n", prefix) - return - } fmt.Printf("%s %s\n", prefix, tn.Hash().StringLE()) case *BranchNode: for i, c := range tn.Children { diff --git a/pkg/core/mpt/branch.go b/pkg/core/mpt/branch.go index d66679d8f..5d4c6ad23 100644 --- a/pkg/core/mpt/branch.go +++ b/pkg/core/mpt/branch.go @@ -27,7 +27,7 @@ var _ Node = (*BranchNode)(nil) func NewBranchNode() *BranchNode { b := new(BranchNode) for i := 0; i < childrenCount; i++ { - b.Children[i] = new(HashNode) + b.Children[i] = EmptyNode{} } return b } diff --git a/pkg/core/mpt/empty.go b/pkg/core/mpt/empty.go new file mode 100644 index 000000000..5d5c3f32c --- /dev/null +++ b/pkg/core/mpt/empty.go @@ -0,0 +1,53 @@ +package mpt + +import ( + "encoding/json" + "errors" + + "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/util" +) + +// EmptyNode represents empty node. +type EmptyNode struct{} + +// DecodeBinary implements io.Serializable interface. +func (e EmptyNode) DecodeBinary(*io.BinReader) { +} + +// EncodeBinary implements io.Serializable interface. +func (e EmptyNode) EncodeBinary(*io.BinWriter) { +} + +// MarshalJSON implements Node interface. +func (e EmptyNode) MarshalJSON() ([]byte, error) { + return []byte(`{}`), nil +} + +// UnmarshalJSON implements Node interface. +func (e EmptyNode) UnmarshalJSON(bytes []byte) error { + var m map[string]interface{} + err := json.Unmarshal(bytes, &m) + if err != nil { + return err + } + if len(m) != 0 { + return errors.New("expected empty node") + } + return nil +} + +// Hash implements Node interface. +func (e EmptyNode) Hash() util.Uint256 { + panic("can't get hash of an EmptyNode") +} + +// Type implements Node interface. +func (e EmptyNode) Type() NodeType { + return EmptyT +} + +// Bytes implements Node interface. +func (e EmptyNode) Bytes() []byte { + return nil +} diff --git a/pkg/core/mpt/hash.go b/pkg/core/mpt/hash.go index f34b542f2..df9ab6017 100644 --- a/pkg/core/mpt/hash.go +++ b/pkg/core/mpt/hash.go @@ -35,9 +35,6 @@ func (h *HashNode) Hash() util.Uint256 { return h.hash } -// IsEmpty returns true if h is an empty node i.e. contains no hash. -func (h *HashNode) IsEmpty() bool { return !h.hashValid } - // Bytes returns serialized HashNode. func (h *HashNode) Bytes() []byte { return h.getBytes(h) @@ -60,9 +57,6 @@ func (h HashNode) EncodeBinary(w *io.BinWriter) { // MarshalJSON implements json.Marshaler. func (h *HashNode) MarshalJSON() ([]byte, error) { - if !h.hashValid { - return []byte(`{}`), nil - } return []byte(`{"hash":"` + h.hash.StringLE() + `"}`), nil } diff --git a/pkg/core/mpt/node.go b/pkg/core/mpt/node.go index 5f8aed265..2eea1fb4a 100644 --- a/pkg/core/mpt/node.go +++ b/pkg/core/mpt/node.go @@ -68,7 +68,7 @@ func (n *NodeObject) UnmarshalJSON(data []byte) error { switch len(m) { case 0: - n.Node = new(HashNode) + n.Node = EmptyNode{} case 1: if v, ok := m["hash"]; ok { var h util.Uint256 diff --git a/pkg/core/mpt/node_test.go b/pkg/core/mpt/node_test.go index 0af0509a9..ca525ac5b 100644 --- a/pkg/core/mpt/node_test.go +++ b/pkg/core/mpt/node_test.go @@ -78,9 +78,6 @@ func TestNode_Serializable(t *testing.T) { t.Run("Raw", getTestFuncEncode(true, h, new(HashNode))) t.Run("WithType", getTestFuncEncode(true, &NodeObject{h}, new(NodeObject))) }) - t.Run("Empty", func(t *testing.T) { // compare nodes, not hashes - testserdes.EncodeDecodeBinary(t, new(HashNode), new(HashNode)) - }) t.Run("InvalidSize", func(t *testing.T) { buf := io.NewBufBinWriter() buf.BinWriter.WriteBytes(make([]byte, 13)) @@ -111,7 +108,7 @@ func TestInvalidJSON(t *testing.T) { t.Run("InvalidChildrenCount", func(t *testing.T) { var cs [childrenCount + 1]Node for i := range cs { - cs[i] = new(HashNode) + cs[i] = EmptyNode{} } data, err := json.Marshal(cs) require.NoError(t, err) diff --git a/pkg/core/mpt/proof.go b/pkg/core/mpt/proof.go index 2def5ac04..8308a6f63 100644 --- a/pkg/core/mpt/proof.go +++ b/pkg/core/mpt/proof.go @@ -49,13 +49,11 @@ func (t *Trie) getProof(curr Node, path []byte, proofs *[][]byte) (Node, error) return n, nil } case *HashNode: - if !n.IsEmpty() { - r, err := t.getFromStore(n.Hash()) - if err != nil { - return nil, err - } - return t.getProof(r, path, proofs) + r, err := t.getFromStore(n.Hash()) + if err != nil { + return nil, err } + return t.getProof(r, path, proofs) } return nil, ErrNotFound } diff --git a/pkg/core/mpt/trie.go b/pkg/core/mpt/trie.go index a59f925e7..7bc9e39d8 100644 --- a/pkg/core/mpt/trie.go +++ b/pkg/core/mpt/trie.go @@ -35,7 +35,7 @@ var ErrNotFound = errors.New("item not found") // This also has the benefit, that every `Put` can be considered an atomic operation. func NewTrie(root Node, enableRefCount bool, store *storage.MemCachedStore) *Trie { if root == nil { - root = new(HashNode) + root = EmptyNode{} } return &Trie{ @@ -75,11 +75,10 @@ func (t *Trie) getWithPath(curr Node, path []byte) (Node, []byte, error) { } n.Children[i] = r return n, bs, nil + case EmptyNode: case *HashNode: - if !n.IsEmpty() { - if r, err := t.getFromStore(n.hash); err == nil { - return t.getWithPath(r, path) - } + if r, err := t.getFromStore(n.hash); err == nil { + return t.getWithPath(r, path) } case *ExtensionNode: if bytes.HasPrefix(path, n.key) { @@ -187,14 +186,13 @@ func (t *Trie) putIntoExtension(curr *ExtensionNode, path []byte, val Node) (Nod return b, nil } +func (t *Trie) putIntoEmpty(path []byte, val Node) (Node, error) { + return t.newSubTrie(path, val, true), nil +} + // putIntoHash puts val to trie if current node is a HashNode. // It returns Node if curr needs to be replaced and error if any. func (t *Trie) putIntoHash(curr *HashNode, path []byte, val Node) (Node, error) { - if curr.IsEmpty() { - hn := t.newSubTrie(path, val, true) - return hn, nil - } - result, err := t.getFromStore(curr.hash) if err != nil { return nil, err @@ -227,6 +225,8 @@ func (t *Trie) putIntoNode(curr Node, path []byte, val Node) (Node, error) { return t.putIntoExtension(n, path, val) case *HashNode: return t.putIntoHash(n, path, val) + case EmptyNode: + return t.putIntoEmpty(path, val) default: panic("invalid MPT node type") } @@ -257,8 +257,7 @@ func (t *Trie) deleteFromBranch(b *BranchNode, path []byte) (Node, error) { b.invalidateCache() var count, index int for i := range b.Children { - h, ok := b.Children[i].(*HashNode) - if !ok || !h.IsEmpty() { + if !isEmpty(b.Children[i]) { index = i count++ } @@ -307,10 +306,9 @@ func (t *Trie) deleteFromExtension(n *ExtensionNode, path []byte) (Node, error) t.removeRef(nxt.Hash(), nxt.bytes) n.key = append(n.key, nxt.key...) n.next = nxt.next + case EmptyNode: + return nxt, nil case *HashNode: - if nxt.IsEmpty() { - return nxt, nil - } n.next = nxt default: n.next = r @@ -327,17 +325,16 @@ func (t *Trie) deleteFromNode(curr Node, path []byte) (Node, error) { case *LeafNode: if len(path) == 0 { t.removeRef(curr.Hash(), curr.Bytes()) - return new(HashNode), nil + return EmptyNode{}, nil } return curr, nil case *BranchNode: return t.deleteFromBranch(n, path) case *ExtensionNode: return t.deleteFromExtension(n, path) + case EmptyNode: + return n, nil case *HashNode: - if n.IsEmpty() { - return n, nil - } newNode, err := t.getFromStore(n.Hash()) if err != nil { return nil, err @@ -350,7 +347,7 @@ func (t *Trie) deleteFromNode(curr Node, path []byte) (Node, error) { // StateRoot returns root hash of t. func (t *Trie) StateRoot() util.Uint256 { - if hn, ok := t.root.(*HashNode); ok && hn.IsEmpty() { + if isEmpty(t.root) { return util.Uint256{} } return t.root.Hash() @@ -486,9 +483,11 @@ func (t *Trie) Collapse(depth int) { } func collapse(depth int, node Node) Node { - if _, ok := node.(*HashNode); ok { + switch node.(type) { + case *HashNode, EmptyNode: return node - } else if depth == 0 { + } + if depth == 0 { return NewHashNode(node.Hash()) } diff --git a/pkg/core/mpt/trie_test.go b/pkg/core/mpt/trie_test.go index 25ddd561c..79fe89551 100644 --- a/pkg/core/mpt/trie_test.go +++ b/pkg/core/mpt/trie_test.go @@ -239,8 +239,7 @@ func isValid(curr Node) bool { if !isValid(n.Children[i]) { return false } - hn, ok := n.Children[i].(*HashNode) - if !ok || !hn.IsEmpty() { + if !isEmpty(n.Children[i]) { count++ } } @@ -342,7 +341,7 @@ func testTrieDelete(t *testing.T, enableGC bool) { }) require.NoError(t, tr.Delete([]byte{0xAB})) - require.True(t, tr.root.(*HashNode).IsEmpty()) + require.IsType(t, EmptyNode{}, tr.root) }) t.Run("MultipleKeys", func(t *testing.T) { @@ -505,12 +504,11 @@ func TestTrie_Collapse(t *testing.T) { require.Equal(t, NewLeafNode([]byte("value")), tr.root) }) t.Run("Hash", func(t *testing.T) { - t.Run("Empty", func(t *testing.T) { - tr := NewTrie(new(HashNode), false, newTestStore()) + t.Run("EmptyNode", func(t *testing.T) { + tr := NewTrie(EmptyNode{}, false, newTestStore()) require.NotPanics(t, func() { tr.Collapse(1) }) - hn, ok := tr.root.(*HashNode) + _, ok := tr.root.(EmptyNode) require.True(t, ok) - require.True(t, hn.IsEmpty()) }) h := random.Uint256() From bd2b1a0521c02826ab18718b7003472c6adc91d5 Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Tue, 3 Aug 2021 18:50:13 +0300 Subject: [PATCH 5/5] mpt: add `Size` method to trie nodes Knowing serialized size of the node is useful for preallocating byte-slice in advance. Signed-off-by: Evgeniy Stratonikov --- pkg/core/mpt/base.go | 6 ++++-- pkg/core/mpt/branch.go | 11 +++++++++++ pkg/core/mpt/empty.go | 3 +++ pkg/core/mpt/extension.go | 6 ++++++ pkg/core/mpt/hash.go | 5 +++++ pkg/core/mpt/leaf.go | 5 +++++ pkg/core/mpt/node.go | 1 + pkg/core/mpt/node_test.go | 1 + 8 files changed, 36 insertions(+), 2 deletions(-) diff --git a/pkg/core/mpt/base.go b/pkg/core/mpt/base.go index f1cd5864d..aa5336880 100644 --- a/pkg/core/mpt/base.go +++ b/pkg/core/mpt/base.go @@ -1,6 +1,7 @@ package mpt import ( + "bytes" "fmt" "github.com/nspcc-dev/neo-go/pkg/crypto/hash" @@ -63,8 +64,9 @@ func (b *BaseNode) updateHash(n Node) { // updateCache updates hash and bytes fields for this BaseNode. func (b *BaseNode) updateBytes(n Node) { - buf := io.NewBufBinWriter() - encodeNodeWithType(n, buf.BinWriter) + buf := bytes.NewBuffer(make([]byte, 0, 1+n.Size())) + bw := io.NewBinWriterFromIO(buf) + encodeNodeWithType(n, bw) b.bytes = buf.Bytes() b.bytesValid = true } diff --git a/pkg/core/mpt/branch.go b/pkg/core/mpt/branch.go index 5d4c6ad23..0338ff4a7 100644 --- a/pkg/core/mpt/branch.go +++ b/pkg/core/mpt/branch.go @@ -45,6 +45,17 @@ func (b *BranchNode) Bytes() []byte { return b.getBytes(b) } +// Size implements Node interface. +func (b *BranchNode) Size() int { + sz := childrenCount + for i := range b.Children { + if !isEmpty(b.Children[i]) { + sz += util.Uint256Size + } + } + return sz +} + // EncodeBinary implements io.Serializable. func (b *BranchNode) EncodeBinary(w *io.BinWriter) { for i := 0; i < childrenCount; i++ { diff --git a/pkg/core/mpt/empty.go b/pkg/core/mpt/empty.go index 5d5c3f32c..6669ef8c1 100644 --- a/pkg/core/mpt/empty.go +++ b/pkg/core/mpt/empty.go @@ -19,6 +19,9 @@ func (e EmptyNode) DecodeBinary(*io.BinReader) { func (e EmptyNode) EncodeBinary(*io.BinWriter) { } +// Size implements Node interface. +func (EmptyNode) Size() int { return 0 } + // MarshalJSON implements Node interface. func (e EmptyNode) MarshalJSON() ([]byte, error) { return []byte(`{}`), nil diff --git a/pkg/core/mpt/extension.go b/pkg/core/mpt/extension.go index 1b8047e20..2dcbcb66b 100644 --- a/pkg/core/mpt/extension.go +++ b/pkg/core/mpt/extension.go @@ -72,6 +72,12 @@ func (e ExtensionNode) EncodeBinary(w *io.BinWriter) { encodeBinaryAsChild(e.next, w) } +// Size implements Node interface. +func (e *ExtensionNode) Size() int { + return io.GetVarSize(len(e.key)) + len(e.key) + + 1 + util.Uint256Size // e.next is never empty +} + // MarshalJSON implements json.Marshaler. func (e *ExtensionNode) MarshalJSON() ([]byte, error) { m := map[string]interface{}{ diff --git a/pkg/core/mpt/hash.go b/pkg/core/mpt/hash.go index df9ab6017..05ddbe5f3 100644 --- a/pkg/core/mpt/hash.go +++ b/pkg/core/mpt/hash.go @@ -27,6 +27,11 @@ func NewHashNode(h util.Uint256) *HashNode { // Type implements Node interface. func (h *HashNode) Type() NodeType { return HashT } +// Size implements Node interface. +func (h *HashNode) Size() int { + return util.Uint256Size +} + // Hash implements Node interface. func (h *HashNode) Hash() util.Uint256 { if !h.hashValid { diff --git a/pkg/core/mpt/leaf.go b/pkg/core/mpt/leaf.go index ecb003c23..0f3072b85 100644 --- a/pkg/core/mpt/leaf.go +++ b/pkg/core/mpt/leaf.go @@ -56,6 +56,11 @@ func (n LeafNode) EncodeBinary(w *io.BinWriter) { w.WriteVarBytes(n.value) } +// Size implements Node interface. +func (n *LeafNode) Size() int { + return io.GetVarSize(len(n.value)) + len(n.value) +} + // MarshalJSON implements json.Marshaler. func (n *LeafNode) MarshalJSON() ([]byte, error) { return []byte(`{"value":"` + hex.EncodeToString(n.value) + `"}`), nil diff --git a/pkg/core/mpt/node.go b/pkg/core/mpt/node.go index 2eea1fb4a..af35286c1 100644 --- a/pkg/core/mpt/node.go +++ b/pkg/core/mpt/node.go @@ -33,6 +33,7 @@ type Node interface { io.Serializable json.Marshaler json.Unmarshaler + Size() int BaseNodeIface } diff --git a/pkg/core/mpt/node_test.go b/pkg/core/mpt/node_test.go index ca525ac5b..a4425d71c 100644 --- a/pkg/core/mpt/node_test.go +++ b/pkg/core/mpt/node_test.go @@ -27,6 +27,7 @@ func getTestFuncEncode(ok bool, expected, actual Node) func(t *testing.T) { require.NoError(t, err) require.Equal(t, expected.Type(), actual.Type()) require.Equal(t, expected.Hash(), actual.Hash()) + require.Equal(t, 1+expected.Size(), len(expected.Bytes())) }) t.Run("JSON", func(t *testing.T) { bs, err := json.Marshal(expected)