From e7687d620db59fc31893ff9faada43246b043bcc Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 6 Dec 2019 17:40:47 +0300 Subject: [PATCH 1/3] io: simplify WriteBytes() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Which speeds it up at least twofold for a typical 32-bytes write (and that's for a very naïve test that allocates new BufBinWriter on every iteration): pkg: github.com/CityOfZion/neo-go/pkg/io BenchmarkWriteBytes-8 10000000 124 ns/op BenchmarkWriteBytesOld-8 5000000 251 ns/op --- pkg/io/binaryWriter.go | 7 +++++-- pkg/io/binaryrw_test.go | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/io/binaryWriter.go b/pkg/io/binaryWriter.go index b5aee0d0c..16e9dfc77 100644 --- a/pkg/io/binaryWriter.go +++ b/pkg/io/binaryWriter.go @@ -95,13 +95,16 @@ func (w *BinWriter) WriteVarUint(val uint64) { // WriteBytes writes a variable byte into the underlying io.Writer without prefix. func (w *BinWriter) WriteBytes(b []byte) { - w.WriteLE(b) + if w.Err != nil { + return + } + _, w.Err = w.w.Write(b) } // WriteVarBytes writes a variable length byte array into the underlying io.Writer. func (w *BinWriter) WriteVarBytes(b []byte) { w.WriteVarUint(uint64(len(b))) - w.WriteLE(b) + w.WriteBytes(b) } // WriteString writes a variable length string into the underlying io.Writer. diff --git a/pkg/io/binaryrw_test.go b/pkg/io/binaryrw_test.go index 6a6278e1e..19a8c2a55 100644 --- a/pkg/io/binaryrw_test.go +++ b/pkg/io/binaryrw_test.go @@ -210,6 +210,11 @@ func TestWriteBytes(t *testing.T) { buf := bw.Bytes() assert.Equal(t, 4, len(buf)) assert.Equal(t, byte(0xde), buf[0]) + + bw = NewBufBinWriter() + bw.Err = errors.New("smth bad") + bw.WriteBytes(bin) + assert.Equal(t, 0, bw.Len()) } type testSerializable uint16 From 844491d36598b9af2e4de08e52b312c43eb01b78 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 6 Dec 2019 18:22:21 +0300 Subject: [PATCH 2/3] *: use more efficient WriteBytes where appropriate Before this patch on block import we could easily be spending more than 6 seconds out of 30 in Uint256 encoding for UnspentBalance, now it's completely off the radar. --- pkg/compiler/emit.go | 2 +- pkg/consensus/commit.go | 2 +- pkg/consensus/payload.go | 4 ++-- pkg/consensus/prepare_request.go | 2 +- pkg/consensus/prepare_response.go | 2 +- pkg/consensus/recovery_message.go | 4 ++-- pkg/core/account_state.go | 4 ++-- pkg/core/asset_state.go | 8 ++++---- pkg/core/block.go | 4 ++-- pkg/core/block_base.go | 8 ++++---- pkg/core/header.go | 2 +- pkg/core/notification_event.go | 4 ++-- pkg/core/spent_coin_state.go | 2 +- pkg/core/transaction/input.go | 2 +- pkg/core/transaction/output.go | 4 ++-- pkg/core/transaction/register.go | 2 +- pkg/core/unspent_coin_state.go | 2 +- pkg/network/message.go | 2 +- pkg/network/payload/address.go | 2 +- pkg/network/payload/getblocks.go | 2 +- pkg/smartcontract/param_context.go | 2 +- pkg/util/uint256.go | 2 +- pkg/vm/serialization.go | 12 ++++++------ 23 files changed, 40 insertions(+), 40 deletions(-) diff --git a/pkg/compiler/emit.go b/pkg/compiler/emit.go index a73b59475..56cffb237 100644 --- a/pkg/compiler/emit.go +++ b/pkg/compiler/emit.go @@ -19,7 +19,7 @@ func emit(w *io.BinWriter, instr opcode.Opcode, b []byte) { // emitOpcode emits a single VM Instruction the given buffer. func emitOpcode(w *io.BinWriter, instr opcode.Opcode) { - w.WriteLE(byte(instr)) + w.WriteBytes([]byte{byte(instr)}) } // emitBool emits a bool type the given buffer. diff --git a/pkg/consensus/commit.go b/pkg/consensus/commit.go index e5d0e0fb9..da20be00f 100644 --- a/pkg/consensus/commit.go +++ b/pkg/consensus/commit.go @@ -18,7 +18,7 @@ var _ payload.Commit = (*commit)(nil) // EncodeBinary implements io.Serializable interface. func (c *commit) EncodeBinary(w *io.BinWriter) { - w.WriteBE(c.signature) + w.WriteBytes(c.signature[:]) } // DecodeBinary implements io.Serializable interface. diff --git a/pkg/consensus/payload.go b/pkg/consensus/payload.go index fce529568..f7ad50a07 100644 --- a/pkg/consensus/payload.go +++ b/pkg/consensus/payload.go @@ -162,7 +162,7 @@ func (p *Payload) SetHeight(h uint32) { // EncodeBinaryUnsigned writes payload to w excluding signature. func (p Payload) EncodeBinaryUnsigned(w *io.BinWriter) { w.WriteLE(p.version) - w.WriteBE(p.prevHash[:]) + w.WriteBytes(p.prevHash[:]) w.WriteLE(p.height) w.WriteLE(p.validatorIndex) w.WriteLE(p.timestamp) @@ -254,7 +254,7 @@ func (p *Payload) DecodeBinary(r *io.BinReader) { // EncodeBinary implements io.Serializable interface. func (m *message) EncodeBinary(w *io.BinWriter) { - w.WriteLE(byte(m.Type)) + w.WriteBytes([]byte{byte(m.Type)}) w.WriteLE(m.ViewNumber) m.payload.EncodeBinary(w) } diff --git a/pkg/consensus/prepare_request.go b/pkg/consensus/prepare_request.go index 6966de0f5..ef98e6512 100644 --- a/pkg/consensus/prepare_request.go +++ b/pkg/consensus/prepare_request.go @@ -22,7 +22,7 @@ var _ payload.PrepareRequest = (*prepareRequest)(nil) func (p *prepareRequest) EncodeBinary(w *io.BinWriter) { w.WriteLE(p.timestamp) w.WriteLE(p.nonce) - w.WriteBE(p.nextConsensus[:]) + w.WriteBytes(p.nextConsensus[:]) w.WriteArray(p.transactionHashes) p.minerTx.EncodeBinary(w) } diff --git a/pkg/consensus/prepare_response.go b/pkg/consensus/prepare_response.go index 8d338bc62..760b9475c 100644 --- a/pkg/consensus/prepare_response.go +++ b/pkg/consensus/prepare_response.go @@ -15,7 +15,7 @@ var _ payload.PrepareResponse = (*prepareResponse)(nil) // EncodeBinary implements io.Serializable interface. func (p *prepareResponse) EncodeBinary(w *io.BinWriter) { - w.WriteBE(p.preparationHash[:]) + w.WriteBytes(p.preparationHash[:]) } // DecodeBinary implements io.Serializable interface. diff --git a/pkg/consensus/recovery_message.go b/pkg/consensus/recovery_message.go index 3a01a168e..aaa1a91f6 100644 --- a/pkg/consensus/recovery_message.go +++ b/pkg/consensus/recovery_message.go @@ -79,7 +79,7 @@ func (m *recoveryMessage) EncodeBinary(w *io.BinWriter) { w.WriteVarUint(0) } else { w.WriteVarUint(util.Uint256Size) - w.WriteBE(m.preparationHash[:]) + w.WriteBytes(m.preparationHash[:]) } } @@ -115,7 +115,7 @@ func (p *commitCompact) DecodeBinary(r *io.BinReader) { func (p *commitCompact) EncodeBinary(w *io.BinWriter) { w.WriteLE(p.ViewNumber) w.WriteLE(p.ValidatorIndex) - w.WriteBE(p.Signature) + w.WriteBytes(p.Signature[:]) w.WriteVarBytes(p.InvocationScript) } diff --git a/pkg/core/account_state.go b/pkg/core/account_state.go index f2779340c..30ca2bc35 100644 --- a/pkg/core/account_state.go +++ b/pkg/core/account_state.go @@ -121,13 +121,13 @@ func (s *AccountState) DecodeBinary(br *io.BinReader) { // EncodeBinary encodes AccountState to the given BinWriter. func (s *AccountState) EncodeBinary(bw *io.BinWriter) { bw.WriteLE(s.Version) - bw.WriteLE(s.ScriptHash) + bw.WriteBytes(s.ScriptHash[:]) bw.WriteLE(s.IsFrozen) bw.WriteArray(s.Votes) bw.WriteVarUint(uint64(len(s.Balances))) for k, v := range s.Balances { - bw.WriteLE(k) + bw.WriteBytes(k[:]) bw.WriteArray(v) } } diff --git a/pkg/core/asset_state.go b/pkg/core/asset_state.go index 8f883a467..10dd5902a 100644 --- a/pkg/core/asset_state.go +++ b/pkg/core/asset_state.go @@ -73,19 +73,19 @@ func (a *AssetState) DecodeBinary(br *io.BinReader) { // EncodeBinary implements Serializable interface. func (a *AssetState) EncodeBinary(bw *io.BinWriter) { - bw.WriteLE(a.ID) + bw.WriteBytes(a.ID[:]) bw.WriteLE(a.AssetType) bw.WriteString(a.Name) bw.WriteLE(a.Amount) bw.WriteLE(a.Available) bw.WriteLE(a.Precision) bw.WriteLE(a.FeeMode) - bw.WriteLE(a.FeeAddress) + bw.WriteBytes(a.FeeAddress[:]) a.Owner.EncodeBinary(bw) - bw.WriteLE(a.Admin) - bw.WriteLE(a.Issuer) + bw.WriteBytes(a.Admin[:]) + bw.WriteBytes(a.Issuer[:]) bw.WriteLE(a.Expiration) bw.WriteLE(a.IsFrozen) } diff --git a/pkg/core/block.go b/pkg/core/block.go index f02fb425f..2dabbcba5 100644 --- a/pkg/core/block.go +++ b/pkg/core/block.go @@ -111,12 +111,12 @@ func NewBlockFromTrimmedBytes(b []byte) (*Block, error) { func (b *Block) Trim() ([]byte, error) { buf := io.NewBufBinWriter() b.encodeHashableFields(buf.BinWriter) - buf.WriteLE(uint8(1)) + buf.WriteBytes([]byte{1}) b.Script.EncodeBinary(buf.BinWriter) buf.WriteVarUint(uint64(len(b.Transactions))) for _, tx := range b.Transactions { - buf.WriteLE(tx.Hash()) + tx.Hash().EncodeBinary(buf.BinWriter) } if buf.Err != nil { return nil, buf.Err diff --git a/pkg/core/block_base.go b/pkg/core/block_base.go index dfbc6cd15..e4a4bd64f 100644 --- a/pkg/core/block_base.go +++ b/pkg/core/block_base.go @@ -87,7 +87,7 @@ func (b *BlockBase) DecodeBinary(br *io.BinReader) { // EncodeBinary implements Serializable interface func (b *BlockBase) EncodeBinary(bw *io.BinWriter) { b.encodeHashableFields(bw) - bw.WriteLE(uint8(1)) + bw.WriteBytes([]byte{1}) b.Script.EncodeBinary(bw) } @@ -116,12 +116,12 @@ func (b *BlockBase) createHash() { // see Hash() for more information about the fields. func (b *BlockBase) encodeHashableFields(bw *io.BinWriter) { bw.WriteLE(b.Version) - bw.WriteLE(b.PrevHash) - bw.WriteLE(b.MerkleRoot) + bw.WriteBytes(b.PrevHash[:]) + bw.WriteBytes(b.MerkleRoot[:]) bw.WriteLE(b.Timestamp) bw.WriteLE(b.Index) bw.WriteLE(b.ConsensusData) - bw.WriteLE(b.NextConsensus) + bw.WriteBytes(b.NextConsensus[:]) } // decodeHashableFields decodes the fields used for hashing. diff --git a/pkg/core/header.go b/pkg/core/header.go index a33f0a0af..e21ce70ed 100644 --- a/pkg/core/header.go +++ b/pkg/core/header.go @@ -29,5 +29,5 @@ func (h *Header) DecodeBinary(r *io.BinReader) { // EncodeBinary implements Serializable interface. func (h *Header) EncodeBinary(w *io.BinWriter) { h.BlockBase.EncodeBinary(w) - w.WriteLE(uint8(0)) + w.WriteBytes([]byte{0}) } diff --git a/pkg/core/notification_event.go b/pkg/core/notification_event.go index 850642557..63774ee7d 100644 --- a/pkg/core/notification_event.go +++ b/pkg/core/notification_event.go @@ -57,7 +57,7 @@ func getAppExecResultFromStore(s storage.Store, hash util.Uint256) (*AppExecResu // EncodeBinary implements the Serializable interface. func (ne NotificationEvent) EncodeBinary(w *io.BinWriter) { - w.WriteLE(ne.ScriptHash) + w.WriteBytes(ne.ScriptHash[:]) vm.EncodeBinaryStackItem(ne.Item, w) } @@ -69,7 +69,7 @@ func (ne *NotificationEvent) DecodeBinary(r *io.BinReader) { // EncodeBinary implements the Serializable interface. func (aer *AppExecResult) EncodeBinary(w *io.BinWriter) { - w.WriteLE(aer.TxHash) + w.WriteBytes(aer.TxHash[:]) w.WriteArray(aer.Events) } diff --git a/pkg/core/spent_coin_state.go b/pkg/core/spent_coin_state.go index 84973df26..f72943707 100644 --- a/pkg/core/spent_coin_state.go +++ b/pkg/core/spent_coin_state.go @@ -93,7 +93,7 @@ func (s *SpentCoinState) DecodeBinary(br *io.BinReader) { // EncodeBinary implements Serializable interface. func (s *SpentCoinState) EncodeBinary(bw *io.BinWriter) { - bw.WriteLE(s.txHash) + bw.WriteBytes(s.txHash[:]) bw.WriteLE(s.txHeight) bw.WriteVarUint(uint64(len(s.items))) for k, v := range s.items { diff --git a/pkg/core/transaction/input.go b/pkg/core/transaction/input.go index f5e25dcc7..27d895afc 100644 --- a/pkg/core/transaction/input.go +++ b/pkg/core/transaction/input.go @@ -22,6 +22,6 @@ func (in *Input) DecodeBinary(br *io.BinReader) { // EncodeBinary implements Serializable interface. func (in *Input) EncodeBinary(bw *io.BinWriter) { - bw.WriteLE(in.PrevHash) + bw.WriteBytes(in.PrevHash[:]) bw.WriteLE(in.PrevIndex) } diff --git a/pkg/core/transaction/output.go b/pkg/core/transaction/output.go index d0ad24006..1e31fa92a 100644 --- a/pkg/core/transaction/output.go +++ b/pkg/core/transaction/output.go @@ -42,9 +42,9 @@ func (out *Output) DecodeBinary(br *io.BinReader) { // EncodeBinary implements Serializable interface. func (out *Output) EncodeBinary(bw *io.BinWriter) { - bw.WriteLE(out.AssetID) + bw.WriteBytes(out.AssetID[:]) bw.WriteLE(out.Amount) - bw.WriteLE(out.ScriptHash) + bw.WriteBytes(out.ScriptHash[:]) } // MarshalJSON implements the Marshaler interface. diff --git a/pkg/core/transaction/register.go b/pkg/core/transaction/register.go index e8d754eb3..cccb6f769 100644 --- a/pkg/core/transaction/register.go +++ b/pkg/core/transaction/register.go @@ -50,5 +50,5 @@ func (tx *RegisterTX) EncodeBinary(bw *io.BinWriter) { bw.WriteLE(tx.Amount) bw.WriteLE(tx.Precision) bw.WriteBytes(tx.Owner.Bytes()) - bw.WriteLE(tx.Admin) + bw.WriteBytes(tx.Admin[:]) } diff --git a/pkg/core/unspent_coin_state.go b/pkg/core/unspent_coin_state.go index a7b566fd8..dd391c66a 100644 --- a/pkg/core/unspent_coin_state.go +++ b/pkg/core/unspent_coin_state.go @@ -92,7 +92,7 @@ func (u UnspentCoins) commit(store storage.Store) error { func (s *UnspentCoinState) EncodeBinary(bw *io.BinWriter) { bw.WriteVarUint(uint64(len(s.states))) for _, state := range s.states { - bw.WriteLE(byte(state)) + bw.WriteBytes([]byte{byte(state)}) } } diff --git a/pkg/network/message.go b/pkg/network/message.go index 2807e22de..9459079ff 100644 --- a/pkg/network/message.go +++ b/pkg/network/message.go @@ -213,7 +213,7 @@ func (m *Message) decodePayload(br *io.BinReader) error { // Encode encodes a Message to any given BinWriter. func (m *Message) Encode(br *io.BinWriter) error { br.WriteLE(m.Magic) - br.WriteLE(m.Command) + br.WriteBytes(m.Command[:]) br.WriteLE(m.Length) br.WriteLE(m.Checksum) if m.Payload != nil { diff --git a/pkg/network/payload/address.go b/pkg/network/payload/address.go index 0daee4456..c8a4f9ece 100644 --- a/pkg/network/payload/address.go +++ b/pkg/network/payload/address.go @@ -39,7 +39,7 @@ func (p *AddressAndTime) DecodeBinary(br *io.BinReader) { func (p *AddressAndTime) EncodeBinary(bw *io.BinWriter) { bw.WriteLE(p.Timestamp) bw.WriteLE(p.Services) - bw.WriteBE(p.IP) + bw.WriteBytes(p.IP[:]) bw.WriteBE(p.Port) } diff --git a/pkg/network/payload/getblocks.go b/pkg/network/payload/getblocks.go index 8701a1df0..5c5caaebc 100644 --- a/pkg/network/payload/getblocks.go +++ b/pkg/network/payload/getblocks.go @@ -30,5 +30,5 @@ func (p *GetBlocks) DecodeBinary(br *io.BinReader) { // EncodeBinary implements Serializable interface. func (p *GetBlocks) EncodeBinary(bw *io.BinWriter) { bw.WriteArray(p.HashStart) - bw.WriteLE(p.HashStop) + bw.WriteBytes(p.HashStop[:]) } diff --git a/pkg/smartcontract/param_context.go b/pkg/smartcontract/param_context.go index ad509f17d..97158844b 100644 --- a/pkg/smartcontract/param_context.go +++ b/pkg/smartcontract/param_context.go @@ -80,7 +80,7 @@ func (pt ParamType) MarshalJSON() ([]byte, error) { // EncodeBinary implements io.Serializable interface. func (pt ParamType) EncodeBinary(w *io.BinWriter) { - w.WriteLE(pt) + w.WriteBytes([]byte{byte(pt)}) } // DecodeBinary implements io.Serializable interface. diff --git a/pkg/util/uint256.go b/pkg/util/uint256.go index 76d69c468..0f3a20fbf 100644 --- a/pkg/util/uint256.go +++ b/pkg/util/uint256.go @@ -98,7 +98,7 @@ func (u Uint256) CompareTo(other Uint256) int { return bytes.Compare(u[:], other // EncodeBinary implements io.Serializable interface. func (u Uint256) EncodeBinary(w *io.BinWriter) { - w.WriteBE(u) + w.WriteBytes(u[:]) } // DecodeBinary implements io.Serializable interface. diff --git a/pkg/vm/serialization.go b/pkg/vm/serialization.go index 5434e8b5a..8f155a8d6 100644 --- a/pkg/vm/serialization.go +++ b/pkg/vm/serialization.go @@ -43,13 +43,13 @@ func serializeItemTo(item StackItem, w *io.BinWriter, seen map[StackItem]bool) { switch t := item.(type) { case *ByteArrayItem: - w.WriteLE(byte(byteArrayT)) + w.WriteBytes([]byte{byte(byteArrayT)}) w.WriteVarBytes(t.value) case *BoolItem: - w.WriteLE(byte(booleanT)) + w.WriteBytes([]byte{byte(booleanT)}) w.WriteLE(t.value) case *BigIntegerItem: - w.WriteLE(byte(integerT)) + w.WriteBytes([]byte{byte(integerT)}) w.WriteVarBytes(t.Bytes()) case *InteropItem: w.Err = errors.New("not supported") @@ -58,9 +58,9 @@ func serializeItemTo(item StackItem, w *io.BinWriter, seen map[StackItem]bool) { _, isArray := t.(*ArrayItem) if isArray { - w.WriteLE(byte(arrayT)) + w.WriteBytes([]byte{byte(arrayT)}) } else { - w.WriteLE(byte(structT)) + w.WriteBytes([]byte{byte(structT)}) } arr := t.Value().([]StackItem) @@ -71,7 +71,7 @@ func serializeItemTo(item StackItem, w *io.BinWriter, seen map[StackItem]bool) { case *MapItem: seen[item] = true - w.WriteLE(byte(mapT)) + w.WriteBytes([]byte{byte(mapT)}) w.WriteVarUint(uint64(len(t.value))) for k, v := range t.value { serializeItemTo(v, w, seen) From 9992a980076351e70fd094cd995243ebd6181826 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 6 Dec 2019 18:44:28 +0300 Subject: [PATCH 3/3] core: optimize balance utxo removal One of my samples had 8 (out of 30) seconds spent here, but values of 100ms are more typical. After this change it becomes invisible. --- pkg/core/blockchain.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 87d219125..bb84c083e 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -402,16 +402,17 @@ func (bc *Blockchain) storeBlock(block *Block) error { if balancesLen <= 1 { delete(account.Balances, prevTXOutput.AssetID) } else { - var gotTx bool - for index, balance := range account.Balances[prevTXOutput.AssetID] { - if !gotTx && balance.Tx.Equals(input.PrevHash) && balance.Index == input.PrevIndex { - gotTx = true - } - if gotTx && index+1 < balancesLen { - account.Balances[prevTXOutput.AssetID][index] = account.Balances[prevTXOutput.AssetID][index+1] + var index = -1 + for i, balance := range account.Balances[prevTXOutput.AssetID] { + if balance.Tx.Equals(input.PrevHash) && balance.Index == input.PrevIndex { + index = i + break } } - account.Balances[prevTXOutput.AssetID] = account.Balances[prevTXOutput.AssetID][:balancesLen-1] + if index >= 0 { + copy(account.Balances[prevTXOutput.AssetID][index:], account.Balances[prevTXOutput.AssetID][index+1:]) + account.Balances[prevTXOutput.AssetID] = account.Balances[prevTXOutput.AssetID][:balancesLen-1] + } } } }