From fe4916f691fc3d23858b4fdd19f51a6fbc2b8465 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 6 Mar 2020 19:26:39 +0300 Subject: [PATCH 1/7] core: simplify GetTransactionResults() a bit Make less movements. --- pkg/core/blockchain.go | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 3fbd5cd20..0bf2d677b 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -1641,36 +1641,24 @@ func (bc *Blockchain) verifyResults(t *transaction.Transaction) error { // GetTransactionResults returns the transaction results aggregate by assetID. // Golang of GetTransationResults method in C# (https://github.com/neo-project/neo/blob/master/neo/Network/P2P/Payloads/Transaction.cs#L207) func (bc *Blockchain) GetTransactionResults(t *transaction.Transaction) []*transaction.Result { - var tempResults []*transaction.Result var results []*transaction.Result - tempGroupResult := make(map[util.Uint256]util.Fixed8) + tempResult := make(map[util.Uint256]util.Fixed8) references, err := bc.References(t) if err != nil { return nil } for _, inout := range references { - tempResults = append(tempResults, &transaction.Result{ - AssetID: inout.Out.AssetID, - Amount: inout.Out.Amount, - }) + c := tempResult[inout.Out.AssetID] + tempResult[inout.Out.AssetID] = c.Add(inout.Out.Amount) } for _, output := range t.Outputs { - tempResults = append(tempResults, &transaction.Result{ - AssetID: output.AssetID, - Amount: -output.Amount, - }) - } - for _, r := range tempResults { - if amount, ok := tempGroupResult[r.AssetID]; ok { - tempGroupResult[r.AssetID] = amount.Add(r.Amount) - } else { - tempGroupResult[r.AssetID] = r.Amount - } + c := tempResult[output.AssetID] + tempResult[output.AssetID] = c.Sub(output.Amount) } results = []*transaction.Result{} // this assignment is necessary. (Most of the time amount == 0 and results is the empty slice.) - for assetID, amount := range tempGroupResult { + for assetID, amount := range tempResult { if amount != util.Fixed8(0) { results = append(results, &transaction.Result{ AssetID: assetID, From baeaa3dbe6c66e033e10cbdc452c82016a3ca153 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 6 Mar 2020 20:15:01 +0300 Subject: [PATCH 2/7] core: optimize tx verification, only get references and results once Getting references requires DB access and that is expensive. --- pkg/core/blockchain.go | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 0bf2d677b..bfff6e60f 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -1364,7 +1364,12 @@ func (bc *Blockchain) verifyTx(t *transaction.Transaction, block *block.Block) e if err := bc.verifyOutputs(t); err != nil { return errors.Wrap(err, "wrong outputs") } - if err := bc.verifyResults(t); err != nil { + refs, err := bc.References(t) + if err != nil { + return err + } + results := refsAndOutsToResults(refs, t.Outputs) + if err := bc.verifyResults(t, results); err != nil { return err } @@ -1383,7 +1388,7 @@ func (bc *Blockchain) verifyTx(t *transaction.Transaction, block *block.Block) e if bc.dao.IsDoubleClaim(claim) { return errors.New("double claim") } - if err := bc.verifyClaims(t); err != nil { + if err := bc.verifyClaims(t, results); err != nil { return err } case transaction.InvocationType: @@ -1396,10 +1401,9 @@ func (bc *Blockchain) verifyTx(t *transaction.Transaction, block *block.Block) e return bc.verifyTxWitnesses(t, block) } -func (bc *Blockchain) verifyClaims(tx *transaction.Transaction) (err error) { +func (bc *Blockchain) verifyClaims(tx *transaction.Transaction, results []*transaction.Result) (err error) { t := tx.Data.(*transaction.ClaimTX) var result *transaction.Result - results := bc.GetTransactionResults(tx) for i := range results { if results[i].AssetID == UtilityTokenID() { result = results[i] @@ -1581,11 +1585,7 @@ func (bc *Blockchain) verifyOutputs(t *transaction.Transaction) error { return nil } -func (bc *Blockchain) verifyResults(t *transaction.Transaction) error { - results := bc.GetTransactionResults(t) - if results == nil { - return errors.New("tx has no results") - } +func (bc *Blockchain) verifyResults(t *transaction.Transaction, results []*transaction.Result) error { var resultsDestroy []*transaction.Result var resultsIssue []*transaction.Result for _, re := range results { @@ -1641,18 +1641,24 @@ func (bc *Blockchain) verifyResults(t *transaction.Transaction) error { // GetTransactionResults returns the transaction results aggregate by assetID. // Golang of GetTransationResults method in C# (https://github.com/neo-project/neo/blob/master/neo/Network/P2P/Payloads/Transaction.cs#L207) func (bc *Blockchain) GetTransactionResults(t *transaction.Transaction) []*transaction.Result { - var results []*transaction.Result - tempResult := make(map[util.Uint256]util.Fixed8) - references, err := bc.References(t) if err != nil { return nil } + return refsAndOutsToResults(references, t.Outputs) +} + +// mapReferencesToResults returns cumulative results of transaction based in its +// references and outputs. +func refsAndOutsToResults(references []transaction.InOut, outputs []transaction.Output) []*transaction.Result { + var results []*transaction.Result + tempResult := make(map[util.Uint256]util.Fixed8) + for _, inout := range references { c := tempResult[inout.Out.AssetID] tempResult[inout.Out.AssetID] = c.Add(inout.Out.Amount) } - for _, output := range t.Outputs { + for _, output := range outputs { c := tempResult[output.AssetID] tempResult[output.AssetID] = c.Sub(output.Amount) } From 0e2bda4f210ff0bc098e1c49f99297beeed24ca1 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 9 Mar 2020 16:31:05 +0300 Subject: [PATCH 3/7] core: drop txHash from SpentCoinState It's a key for it, makes no sense storing it as data. --- pkg/core/dao.go | 2 +- pkg/core/spent_coin_state.go | 7 +------ pkg/core/spent_coin_state_test.go | 1 - 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/core/dao.go b/pkg/core/dao.go index d64b1426c..4973a3b76 100644 --- a/pkg/core/dao.go +++ b/pkg/core/dao.go @@ -219,7 +219,7 @@ func (dao *dao) GetSpentCoinsOrNew(hash util.Uint256, height uint32) (*SpentCoin if err != storage.ErrKeyNotFound { return nil, err } - spent = NewSpentCoinState(hash, height) + spent = NewSpentCoinState(height) } return spent, nil } diff --git a/pkg/core/spent_coin_state.go b/pkg/core/spent_coin_state.go index b7702f9c6..85f0e4008 100644 --- a/pkg/core/spent_coin_state.go +++ b/pkg/core/spent_coin_state.go @@ -3,12 +3,10 @@ package core import ( "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/io" - "github.com/nspcc-dev/neo-go/pkg/util" ) // SpentCoinState represents the state of a spent coin. type SpentCoinState struct { - txHash util.Uint256 txHeight uint32 // A mapping between the index of the prevIndex and block height. @@ -23,9 +21,8 @@ type spentCoin struct { } // NewSpentCoinState returns a new SpentCoinState object. -func NewSpentCoinState(hash util.Uint256, height uint32) *SpentCoinState { +func NewSpentCoinState(height uint32) *SpentCoinState { return &SpentCoinState{ - txHash: hash, txHeight: height, items: make(map[uint16]uint32), } @@ -33,7 +30,6 @@ func NewSpentCoinState(hash util.Uint256, height uint32) *SpentCoinState { // DecodeBinary implements Serializable interface. func (s *SpentCoinState) DecodeBinary(br *io.BinReader) { - br.ReadBytes(s.txHash[:]) s.txHeight = br.ReadU32LE() s.items = make(map[uint16]uint32) @@ -51,7 +47,6 @@ func (s *SpentCoinState) DecodeBinary(br *io.BinReader) { // EncodeBinary implements Serializable interface. func (s *SpentCoinState) EncodeBinary(bw *io.BinWriter) { - bw.WriteBytes(s.txHash[:]) bw.WriteU32LE(s.txHeight) bw.WriteVarUint(uint64(len(s.items))) for k, v := range s.items { diff --git a/pkg/core/spent_coin_state_test.go b/pkg/core/spent_coin_state_test.go index b6a289974..4fce2b386 100644 --- a/pkg/core/spent_coin_state_test.go +++ b/pkg/core/spent_coin_state_test.go @@ -10,7 +10,6 @@ import ( func TestEncodeDecodeSpentCoinState(t *testing.T) { spent := &SpentCoinState{ - txHash: random.Uint256(), txHeight: 1001, items: map[uint16]uint32{ 1: 3, From 377fb382aab019f90056984afc34b85388db03a0 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 9 Mar 2020 16:56:37 +0300 Subject: [PATCH 4/7] core: move (un)SpentCoin structs into the state package As they're all about the state. --- pkg/core/blockchain.go | 20 +++---- pkg/core/blockchainer.go | 2 +- pkg/core/dao.go | 26 ++++----- pkg/core/dao_test.go | 6 +- pkg/core/spent_coin.go | 10 ++++ pkg/core/spent_coin_state.go | 56 ------------------- pkg/core/state/spent_coin.go | 46 +++++++++++++++ .../spent_coin_test.go} | 13 ++--- pkg/core/state/unspent_coin.go | 38 +++++++++++++ pkg/core/state/unspent_coin_test.go | 28 ++++++++++ pkg/core/unspent_coin_state.go | 39 ------------- pkg/core/unspent_coint_state_test.go | 29 ---------- pkg/network/helper_test.go | 3 +- 13 files changed, 155 insertions(+), 161 deletions(-) create mode 100644 pkg/core/spent_coin.go delete mode 100644 pkg/core/spent_coin_state.go create mode 100644 pkg/core/state/spent_coin.go rename pkg/core/{spent_coin_state_test.go => state/spent_coin_test.go} (61%) create mode 100644 pkg/core/state/unspent_coin.go create mode 100644 pkg/core/state/unspent_coin_test.go delete mode 100644 pkg/core/unspent_coin_state.go delete mode 100644 pkg/core/unspent_coint_state_test.go diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index bfff6e60f..d7bfc42e2 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -475,7 +475,7 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { return err } - if err := cache.PutUnspentCoinState(tx.Hash(), NewUnspentCoinState(len(tx.Outputs))); err != nil { + if err := cache.PutUnspentCoinState(tx.Hash(), state.NewUnspentCoin(len(tx.Outputs))); err != nil { return err } @@ -499,9 +499,9 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { if err != nil { return err } - oldSpentCoinLen := len(spentCoin.items) + oldSpentCoinLen := len(spentCoin.Items) for _, input := range inputs { - unspent.states[input.PrevIndex] = state.CoinSpent + unspent.States[input.PrevIndex] = state.CoinSpent prevTXOutput := prevTX.Outputs[input.PrevIndex] account, err := cache.GetAccountStateOrNew(prevTXOutput.ScriptHash) if err != nil { @@ -516,7 +516,7 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { End: block.Index, Value: prevTXOutput.Amount, }) - spentCoin.items[input.PrevIndex] = block.Index + spentCoin.Items[input.PrevIndex] = block.Index if err = processTXWithValidatorsSubtract(&prevTXOutput, account, cache); err != nil { return err } @@ -548,7 +548,7 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { if err = cache.PutUnspentCoinState(prevHash, unspent); err != nil { return err } - if oldSpentCoinLen != len(spentCoin.items) { + if oldSpentCoinLen != len(spentCoin.Items) { if err = cache.PutSpentCoinState(prevHash, spentCoin); err != nil { return err } @@ -590,7 +590,7 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { for _, input := range t.Claims { scs, err := cache.GetSpentCoinState(input.PrevHash) if err == nil { - _, ok := scs.items[input.PrevIndex] + _, ok := scs.Items[input.PrevIndex] if !ok { err = errors.New("no spent coin state") } @@ -644,8 +644,8 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { return err } - delete(scs.items, input.PrevIndex) - if len(scs.items) > 0 { + delete(scs.Items, input.PrevIndex) + if len(scs.Items) > 0 { if err = cache.PutSpentCoinState(input.PrevHash, scs); err != nil { return err } @@ -1174,7 +1174,7 @@ func (bc *Blockchain) GetAccountState(scriptHash util.Uint160) *state.Account { } // GetUnspentCoinState returns unspent coin state for given tx hash. -func (bc *Blockchain) GetUnspentCoinState(hash util.Uint256) *UnspentCoinState { +func (bc *Blockchain) GetUnspentCoinState(hash util.Uint256) *state.UnspentCoin { ucs, err := bc.dao.GetUnspentCoinState(hash) if ucs == nil && err != storage.ErrKeyNotFound { bc.log.Warn("failed to get unspent coin state", zap.Error(err)) @@ -1472,7 +1472,7 @@ func (bc *Blockchain) getUnclaimed(h util.Uint256) (map[uint16]*spentCoin, error } result := make(map[uint16]*spentCoin) - for i, height := range scs.items { + for i, height := range scs.Items { result[i] = &spentCoin{ Output: &tx.Outputs[i], StartHeight: txHeight, diff --git a/pkg/core/blockchainer.go b/pkg/core/blockchainer.go index 0c2aa0065..eb10527ca 100644 --- a/pkg/core/blockchainer.go +++ b/pkg/core/blockchainer.go @@ -41,7 +41,7 @@ type Blockchainer interface { GetStorageItems(hash util.Uint160) (map[string]*state.StorageItem, error) GetTestVM() (*vm.VM, storage.Store) GetTransaction(util.Uint256) (*transaction.Transaction, uint32, error) - GetUnspentCoinState(util.Uint256) *UnspentCoinState + GetUnspentCoinState(util.Uint256) *state.UnspentCoin References(t *transaction.Transaction) ([]transaction.InOut, error) mempool.Feer // fee interface PoolTx(*transaction.Transaction) error diff --git a/pkg/core/dao.go b/pkg/core/dao.go index 4973a3b76..fbab4c6ef 100644 --- a/pkg/core/dao.go +++ b/pkg/core/dao.go @@ -178,22 +178,20 @@ func (dao *dao) AppendNEP5Transfer(acc util.Uint160, tr *state.NEP5Transfer) err // GetUnspentCoinStateOrNew gets UnspentCoinState from temporary or persistent Store // and return it. If it's not present in both stores, returns a new // UnspentCoinState. -func (dao *dao) GetUnspentCoinStateOrNew(hash util.Uint256) (*UnspentCoinState, error) { +func (dao *dao) GetUnspentCoinStateOrNew(hash util.Uint256) (*state.UnspentCoin, error) { unspent, err := dao.GetUnspentCoinState(hash) if err != nil { if err != storage.ErrKeyNotFound { return nil, err } - unspent = &UnspentCoinState{ - states: []state.Coin{}, - } + unspent = state.NewUnspentCoin(0) } return unspent, nil } // GetUnspentCoinState retrieves UnspentCoinState from the given store. -func (dao *dao) GetUnspentCoinState(hash util.Uint256) (*UnspentCoinState, error) { - unspent := &UnspentCoinState{} +func (dao *dao) GetUnspentCoinState(hash util.Uint256) (*state.UnspentCoin, error) { + unspent := &state.UnspentCoin{} key := storage.AppendPrefix(storage.STCoin, hash.BytesLE()) err := dao.GetAndDecode(unspent, key) if err != nil { @@ -203,7 +201,7 @@ func (dao *dao) GetUnspentCoinState(hash util.Uint256) (*UnspentCoinState, error } // PutUnspentCoinState puts given UnspentCoinState into the given store. -func (dao *dao) PutUnspentCoinState(hash util.Uint256, ucs *UnspentCoinState) error { +func (dao *dao) PutUnspentCoinState(hash util.Uint256, ucs *state.UnspentCoin) error { key := storage.AppendPrefix(storage.STCoin, hash.BytesLE()) return dao.Put(ucs, key) } @@ -213,20 +211,20 @@ func (dao *dao) PutUnspentCoinState(hash util.Uint256, ucs *UnspentCoinState) er // -- start spent coins. // GetSpentCoinsOrNew returns spent coins from store. -func (dao *dao) GetSpentCoinsOrNew(hash util.Uint256, height uint32) (*SpentCoinState, error) { +func (dao *dao) GetSpentCoinsOrNew(hash util.Uint256, height uint32) (*state.SpentCoin, error) { spent, err := dao.GetSpentCoinState(hash) if err != nil { if err != storage.ErrKeyNotFound { return nil, err } - spent = NewSpentCoinState(height) + spent = state.NewSpentCoin(height) } return spent, nil } // GetSpentCoinState gets SpentCoinState from the given store. -func (dao *dao) GetSpentCoinState(hash util.Uint256) (*SpentCoinState, error) { - spent := &SpentCoinState{} +func (dao *dao) GetSpentCoinState(hash util.Uint256) (*state.SpentCoin, error) { + spent := &state.SpentCoin{} key := storage.AppendPrefix(storage.STSpentCoin, hash.BytesLE()) err := dao.GetAndDecode(spent, key) if err != nil { @@ -236,7 +234,7 @@ func (dao *dao) GetSpentCoinState(hash util.Uint256) (*SpentCoinState, error) { } // PutSpentCoinState puts given SpentCoinState into the given store. -func (dao *dao) PutSpentCoinState(hash util.Uint256, scs *SpentCoinState) error { +func (dao *dao) PutSpentCoinState(hash util.Uint256, scs *state.SpentCoin) error { key := storage.AppendPrefix(storage.STSpentCoin, hash.BytesLE()) return dao.Put(scs, key) } @@ -592,7 +590,7 @@ func (dao *dao) IsDoubleSpend(tx *transaction.Transaction) bool { return false } for _, input := range inputs { - if int(input.PrevIndex) >= len(unspent.states) || unspent.states[input.PrevIndex] == state.CoinSpent { + if int(input.PrevIndex) >= len(unspent.States) || unspent.States[input.PrevIndex] == state.CoinSpent { return true } } @@ -612,7 +610,7 @@ func (dao *dao) IsDoubleClaim(claim *transaction.ClaimTX) bool { return true } for _, input := range inputs { - _, ok := scs.items[input.PrevIndex] + _, ok := scs.Items[input.PrevIndex] if !ok { return true } diff --git a/pkg/core/dao_test.go b/pkg/core/dao_test.go index a854265ff..962d16df0 100644 --- a/pkg/core/dao_test.go +++ b/pkg/core/dao_test.go @@ -113,7 +113,7 @@ func TestGetUnspentCoinState_Err(t *testing.T) { func TestPutGetUnspentCoinState(t *testing.T) { dao := newDao(storage.NewMemoryStore()) hash := random.Uint256() - unspentCoinState := &UnspentCoinState{states: []state.Coin{}} + unspentCoinState := &state.UnspentCoin{States: []state.Coin{}} err := dao.PutUnspentCoinState(hash, unspentCoinState) require.NoError(t, err) gotUnspentCoinState, err := dao.GetUnspentCoinState(hash) @@ -132,7 +132,7 @@ func TestGetSpentCoinStateOrNew_New(t *testing.T) { func TestPutAndGetSpentCoinState(t *testing.T) { dao := newDao(storage.NewMemoryStore()) hash := random.Uint256() - spentCoinState := &SpentCoinState{items: make(map[uint16]uint32)} + spentCoinState := &state.SpentCoin{Items: make(map[uint16]uint32)} err := dao.PutSpentCoinState(hash, spentCoinState) require.NoError(t, err) gotSpentCoinState, err := dao.GetSpentCoinState(hash) @@ -151,7 +151,7 @@ func TestGetSpentCoinState_Err(t *testing.T) { func TestDeleteSpentCoinState(t *testing.T) { dao := newDao(storage.NewMemoryStore()) hash := random.Uint256() - spentCoinState := &SpentCoinState{items: make(map[uint16]uint32)} + spentCoinState := &state.SpentCoin{Items: make(map[uint16]uint32)} err := dao.PutSpentCoinState(hash, spentCoinState) require.NoError(t, err) err = dao.DeleteSpentCoinState(hash) diff --git a/pkg/core/spent_coin.go b/pkg/core/spent_coin.go new file mode 100644 index 000000000..9d42cb8af --- /dev/null +++ b/pkg/core/spent_coin.go @@ -0,0 +1,10 @@ +package core + +import "github.com/nspcc-dev/neo-go/pkg/core/transaction" + +// spentCoin represents the state of a single spent coin output. +type spentCoin struct { + Output *transaction.Output + StartHeight uint32 + EndHeight uint32 +} diff --git a/pkg/core/spent_coin_state.go b/pkg/core/spent_coin_state.go deleted file mode 100644 index 85f0e4008..000000000 --- a/pkg/core/spent_coin_state.go +++ /dev/null @@ -1,56 +0,0 @@ -package core - -import ( - "github.com/nspcc-dev/neo-go/pkg/core/transaction" - "github.com/nspcc-dev/neo-go/pkg/io" -) - -// SpentCoinState represents the state of a spent coin. -type SpentCoinState struct { - txHeight uint32 - - // A mapping between the index of the prevIndex and block height. - items map[uint16]uint32 -} - -// spentCoin represents the state of a single spent coin output. -type spentCoin struct { - Output *transaction.Output - StartHeight uint32 - EndHeight uint32 -} - -// NewSpentCoinState returns a new SpentCoinState object. -func NewSpentCoinState(height uint32) *SpentCoinState { - return &SpentCoinState{ - txHeight: height, - items: make(map[uint16]uint32), - } -} - -// DecodeBinary implements Serializable interface. -func (s *SpentCoinState) DecodeBinary(br *io.BinReader) { - s.txHeight = br.ReadU32LE() - - s.items = make(map[uint16]uint32) - lenItems := br.ReadVarUint() - for i := 0; i < int(lenItems); i++ { - var ( - key uint16 - value uint32 - ) - key = br.ReadU16LE() - value = br.ReadU32LE() - s.items[key] = value - } -} - -// EncodeBinary implements Serializable interface. -func (s *SpentCoinState) EncodeBinary(bw *io.BinWriter) { - bw.WriteU32LE(s.txHeight) - bw.WriteVarUint(uint64(len(s.items))) - for k, v := range s.items { - bw.WriteU16LE(k) - bw.WriteU32LE(v) - } -} diff --git a/pkg/core/state/spent_coin.go b/pkg/core/state/spent_coin.go new file mode 100644 index 000000000..d1f2c3553 --- /dev/null +++ b/pkg/core/state/spent_coin.go @@ -0,0 +1,46 @@ +package state + +import "github.com/nspcc-dev/neo-go/pkg/io" + +// SpentCoin represents the state of a spent coin. +type SpentCoin struct { + TxHeight uint32 + + // A mapping between the index of the prevIndex and block height. + Items map[uint16]uint32 +} + +// NewSpentCoin returns a new SpentCoin object. +func NewSpentCoin(height uint32) *SpentCoin { + return &SpentCoin{ + TxHeight: height, + Items: make(map[uint16]uint32), + } +} + +// DecodeBinary implements Serializable interface. +func (s *SpentCoin) DecodeBinary(br *io.BinReader) { + s.TxHeight = br.ReadU32LE() + + s.Items = make(map[uint16]uint32) + lenItems := br.ReadVarUint() + for i := 0; i < int(lenItems); i++ { + var ( + key uint16 + value uint32 + ) + key = br.ReadU16LE() + value = br.ReadU32LE() + s.Items[key] = value + } +} + +// EncodeBinary implements Serializable interface. +func (s *SpentCoin) EncodeBinary(bw *io.BinWriter) { + bw.WriteU32LE(s.TxHeight) + bw.WriteVarUint(uint64(len(s.Items))) + for k, v := range s.Items { + bw.WriteU16LE(k) + bw.WriteU32LE(v) + } +} diff --git a/pkg/core/spent_coin_state_test.go b/pkg/core/state/spent_coin_test.go similarity index 61% rename from pkg/core/spent_coin_state_test.go rename to pkg/core/state/spent_coin_test.go index 4fce2b386..8759a9ba3 100644 --- a/pkg/core/spent_coin_state_test.go +++ b/pkg/core/state/spent_coin_test.go @@ -1,17 +1,16 @@ -package core +package state import ( "testing" - "github.com/nspcc-dev/neo-go/pkg/internal/random" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/stretchr/testify/assert" ) -func TestEncodeDecodeSpentCoinState(t *testing.T) { - spent := &SpentCoinState{ - txHeight: 1001, - items: map[uint16]uint32{ +func TestEncodeDecodeSpentCoin(t *testing.T) { + spent := &SpentCoin{ + TxHeight: 1001, + Items: map[uint16]uint32{ 1: 3, 2: 8, 4: 100, @@ -21,7 +20,7 @@ func TestEncodeDecodeSpentCoinState(t *testing.T) { buf := io.NewBufBinWriter() spent.EncodeBinary(buf.BinWriter) assert.Nil(t, buf.Err) - spentDecode := new(SpentCoinState) + spentDecode := new(SpentCoin) r := io.NewBinReaderFromBuf(buf.Bytes()) spentDecode.DecodeBinary(r) assert.Nil(t, r.Err) diff --git a/pkg/core/state/unspent_coin.go b/pkg/core/state/unspent_coin.go new file mode 100644 index 000000000..d153ef639 --- /dev/null +++ b/pkg/core/state/unspent_coin.go @@ -0,0 +1,38 @@ +package state + +import ( + "github.com/nspcc-dev/neo-go/pkg/io" +) + +// UnspentCoin hold the state of a unspent coin. +type UnspentCoin struct { + States []Coin +} + +// NewUnspentCoin returns a new unspent coin state with N confirmed states. +func NewUnspentCoin(n int) *UnspentCoin { + u := &UnspentCoin{ + States: make([]Coin, n), + } + for i := 0; i < n; i++ { + u.States[i] = CoinConfirmed + } + return u +} + +// EncodeBinary encodes UnspentCoin to the given BinWriter. +func (s *UnspentCoin) EncodeBinary(bw *io.BinWriter) { + bw.WriteVarUint(uint64(len(s.States))) + for _, state := range s.States { + bw.WriteB(byte(state)) + } +} + +// DecodeBinary decodes UnspentCoin from the given BinReader. +func (s *UnspentCoin) DecodeBinary(br *io.BinReader) { + lenStates := br.ReadVarUint() + s.States = make([]Coin, lenStates) + for i := 0; i < int(lenStates); i++ { + s.States[i] = Coin(br.ReadB()) + } +} diff --git a/pkg/core/state/unspent_coin_test.go b/pkg/core/state/unspent_coin_test.go new file mode 100644 index 000000000..85f82e331 --- /dev/null +++ b/pkg/core/state/unspent_coin_test.go @@ -0,0 +1,28 @@ +package state + +import ( + "testing" + + "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/stretchr/testify/assert" +) + +func TestDecodeEncodeUnspentCoin(t *testing.T) { + unspent := &UnspentCoin{ + States: []Coin{ + CoinConfirmed, + CoinSpent, + CoinSpent, + CoinSpent, + CoinConfirmed, + }, + } + + buf := io.NewBufBinWriter() + unspent.EncodeBinary(buf.BinWriter) + assert.Nil(t, buf.Err) + unspentDecode := &UnspentCoin{} + r := io.NewBinReaderFromBuf(buf.Bytes()) + unspentDecode.DecodeBinary(r) + assert.Nil(t, r.Err) +} diff --git a/pkg/core/unspent_coin_state.go b/pkg/core/unspent_coin_state.go deleted file mode 100644 index cbcd6a2e8..000000000 --- a/pkg/core/unspent_coin_state.go +++ /dev/null @@ -1,39 +0,0 @@ -package core - -import ( - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/io" -) - -// UnspentCoinState hold the state of a unspent coin. -type UnspentCoinState struct { - states []state.Coin -} - -// NewUnspentCoinState returns a new unspent coin state with N confirmed states. -func NewUnspentCoinState(n int) *UnspentCoinState { - u := &UnspentCoinState{ - states: make([]state.Coin, n), - } - for i := 0; i < n; i++ { - u.states[i] = state.CoinConfirmed - } - return u -} - -// EncodeBinary encodes UnspentCoinState to the given BinWriter. -func (s *UnspentCoinState) EncodeBinary(bw *io.BinWriter) { - bw.WriteVarUint(uint64(len(s.states))) - for _, state := range s.states { - bw.WriteB(byte(state)) - } -} - -// DecodeBinary decodes UnspentCoinState from the given BinReader. -func (s *UnspentCoinState) DecodeBinary(br *io.BinReader) { - lenStates := br.ReadVarUint() - s.states = make([]state.Coin, lenStates) - for i := 0; i < int(lenStates); i++ { - s.states[i] = state.Coin(br.ReadB()) - } -} diff --git a/pkg/core/unspent_coint_state_test.go b/pkg/core/unspent_coint_state_test.go deleted file mode 100644 index 98e089f58..000000000 --- a/pkg/core/unspent_coint_state_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package core - -import ( - "testing" - - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/io" - "github.com/stretchr/testify/assert" -) - -func TestDecodeEncodeUnspentCoinState(t *testing.T) { - unspent := &UnspentCoinState{ - states: []state.Coin{ - state.CoinConfirmed, - state.CoinSpent, - state.CoinSpent, - state.CoinSpent, - state.CoinConfirmed, - }, - } - - buf := io.NewBufBinWriter() - unspent.EncodeBinary(buf.BinWriter) - assert.Nil(t, buf.Err) - unspentDecode := &UnspentCoinState{} - r := io.NewBinReaderFromBuf(buf.Bytes()) - unspentDecode.DecodeBinary(r) - assert.Nil(t, r.Err) -} diff --git a/pkg/network/helper_test.go b/pkg/network/helper_test.go index ff2edbc4b..28df85ad3 100644 --- a/pkg/network/helper_test.go +++ b/pkg/network/helper_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/nspcc-dev/neo-go/config" - "github.com/nspcc-dev/neo-go/pkg/core" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/mempool" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -126,7 +125,7 @@ func (chain testChain) GetTransaction(util.Uint256) (*transaction.Transaction, u panic("TODO") } -func (chain testChain) GetUnspentCoinState(util.Uint256) *core.UnspentCoinState { +func (chain testChain) GetUnspentCoinState(util.Uint256) *state.UnspentCoin { panic("TODO") } From e1f194ea7b8f8c6a49d647869b2c709c57634e69 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 9 Mar 2020 17:18:04 +0300 Subject: [PATCH 5/7] core: treat state.Coin as a bitfield As it was intended to. --- pkg/core/blockchain.go | 2 +- pkg/core/dao.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index d7bfc42e2..23f2610cc 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -501,7 +501,7 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { } oldSpentCoinLen := len(spentCoin.Items) for _, input := range inputs { - unspent.States[input.PrevIndex] = state.CoinSpent + unspent.States[input.PrevIndex] |= state.CoinSpent prevTXOutput := prevTX.Outputs[input.PrevIndex] account, err := cache.GetAccountStateOrNew(prevTXOutput.ScriptHash) if err != nil { diff --git a/pkg/core/dao.go b/pkg/core/dao.go index fbab4c6ef..f211a2d42 100644 --- a/pkg/core/dao.go +++ b/pkg/core/dao.go @@ -590,7 +590,7 @@ func (dao *dao) IsDoubleSpend(tx *transaction.Transaction) bool { return false } for _, input := range inputs { - if int(input.PrevIndex) >= len(unspent.States) || unspent.States[input.PrevIndex] == state.CoinSpent { + if int(input.PrevIndex) >= len(unspent.States) || (unspent.States[input.PrevIndex]&state.CoinSpent) != 0 { return true } } From 23464401bcdd097d4667eebeb5d45690be41df2c Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 9 Mar 2020 18:56:24 +0300 Subject: [PATCH 6/7] core/state: merge spent and unspent coins state, use it to store more things This change reduces pressure on DB by doing the following things: * not storing additional KV pair for SpentCoin * storing Output right in the UnspentCoin, thus eliminating the need to get a full transaction from DB At the same time it makes UnspentCoin more fat and hot, but it should probably worth it. Also drop `GetUnspentCoinStateOrNew` as it shouldn't ever existed, UTXOs can't come out of nowhere. 1.5M block import time (VerifyBlocks disabled) on AMD Ryzen 5 1600/16GB/HDD, before: real 302m9.895s user 96m17.200s sys 13m37.084s after: real 159m16.551s user 69m58.279s sys 7m34.334s So it's almost two-fold which is a great improvement. --- pkg/core/blockchain.go | 125 ++++++++++------------------ pkg/core/dao.go | 60 +------------ pkg/core/dao_test.go | 50 +---------- pkg/core/state/spent_coin.go | 46 ---------- pkg/core/state/spent_coin_test.go | 28 ------- pkg/core/state/unspent_coin.go | 48 ++++++++--- pkg/core/state/unspent_coin_test.go | 38 +++++++-- 7 files changed, 117 insertions(+), 278 deletions(-) delete mode 100644 pkg/core/state/spent_coin.go delete mode 100644 pkg/core/state/spent_coin_test.go diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 23f2610cc..305058a37 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -29,7 +29,7 @@ import ( // Tuning parameters. const ( headerBatchCount = 2000 - version = "0.0.6" + version = "0.0.7" // This one comes from C# code and it's different from the constant used // when creating an asset with Neo.Asset.Create interop call. It looks @@ -475,7 +475,7 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { return err } - if err := cache.PutUnspentCoinState(tx.Hash(), state.NewUnspentCoin(len(tx.Outputs))); err != nil { + if err := cache.PutUnspentCoinState(tx.Hash(), state.NewUnspentCoin(block.Index, tx)); err != nil { return err } @@ -487,22 +487,20 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { // Process TX inputs that are grouped by previous hash. for _, inputs := range transaction.GroupInputsByPrevHash(tx.Inputs) { prevHash := inputs[0].PrevHash - prevTX, prevTXHeight, err := bc.dao.GetTransaction(prevHash) - if err != nil { - return fmt.Errorf("could not find previous TX: %s", prevHash) - } - unspent, err := cache.GetUnspentCoinStateOrNew(prevHash) + unspent, err := cache.GetUnspentCoinState(prevHash) if err != nil { return err } - spentCoin, err := cache.GetSpentCoinsOrNew(prevHash, prevTXHeight) - if err != nil { - return err - } - oldSpentCoinLen := len(spentCoin.Items) for _, input := range inputs { - unspent.States[input.PrevIndex] |= state.CoinSpent - prevTXOutput := prevTX.Outputs[input.PrevIndex] + if len(unspent.States) <= int(input.PrevIndex) { + return fmt.Errorf("bad input: %s/%d", input.PrevHash.StringLE(), input.PrevIndex) + } + if unspent.States[input.PrevIndex].State&state.CoinSpent != 0 { + return fmt.Errorf("double spend: %s/%d", input.PrevHash.StringLE(), input.PrevIndex) + } + unspent.States[input.PrevIndex].State |= state.CoinSpent + unspent.States[input.PrevIndex].SpendHeight = block.Index + prevTXOutput := &unspent.States[input.PrevIndex].Output account, err := cache.GetAccountStateOrNew(prevTXOutput.ScriptHash) if err != nil { return err @@ -510,14 +508,13 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { if prevTXOutput.AssetID.Equals(GoverningTokenID()) { account.Unclaimed = append(account.Unclaimed, state.UnclaimedBalance{ - Tx: prevTX.Hash(), + Tx: input.PrevHash, Index: input.PrevIndex, - Start: prevTXHeight, + Start: unspent.Height, End: block.Index, Value: prevTXOutput.Amount, }) - spentCoin.Items[input.PrevIndex] = block.Index - if err = processTXWithValidatorsSubtract(&prevTXOutput, account, cache); err != nil { + if err = processTXWithValidatorsSubtract(prevTXOutput, account, cache); err != nil { return err } } @@ -548,11 +545,6 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { if err = cache.PutUnspentCoinState(prevHash, unspent); err != nil { return err } - if oldSpentCoinLen != len(spentCoin.Items) { - if err = cache.PutSpentCoinState(prevHash, spentCoin); err != nil { - return err - } - } } // Process the underlying type of the TX. @@ -588,17 +580,18 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { // Remove claimed NEO from spent coins making it unavalaible for // additional claims. for _, input := range t.Claims { - scs, err := cache.GetSpentCoinState(input.PrevHash) + scs, err := cache.GetUnspentCoinState(input.PrevHash) if err == nil { - _, ok := scs.Items[input.PrevIndex] - if !ok { - err = errors.New("no spent coin state") + if len(scs.States) <= int(input.PrevIndex) { + err = errors.New("invalid claim index") + } else if scs.States[input.PrevIndex].State&state.CoinClaimed != 0 { + err = errors.New("double claim") } } if err != nil { // We can't really do anything about it // as it's a transaction in a signed block. - bc.log.Warn("DOUBLE CLAIM", + bc.log.Warn("FALSE OR DOUBLE CLAIM", zap.String("PrevHash", input.PrevHash.StringLE()), zap.Uint16("PrevIndex", input.PrevIndex), zap.String("tx", tx.Hash().StringLE()), @@ -611,14 +604,13 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { break } - prevTx, _, err := cache.GetTransaction(input.PrevHash) + acc, err := cache.GetAccountState(scs.States[input.PrevIndex].ScriptHash) if err != nil { return err - } else if int(input.PrevIndex) > len(prevTx.Outputs) { - return errors.New("invalid input in claim") } - acc, err := cache.GetAccountState(prevTx.Outputs[input.PrevIndex].ScriptHash) - if err != nil { + + scs.States[input.PrevIndex].State |= state.CoinClaimed + if err = cache.PutUnspentCoinState(input.PrevHash, scs); err != nil { return err } @@ -643,17 +635,6 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { } else if err := cache.PutAccountState(acc); err != nil { return err } - - delete(scs.Items, input.PrevIndex) - if len(scs.Items) > 0 { - if err = cache.PutSpentCoinState(input.PrevHash, scs); err != nil { - return err - } - } else { - if err = cache.DeleteSpentCoinState(input.PrevHash); err != nil { - return err - } - } } case *transaction.EnrollmentTX: if err := processEnrollmentTX(cache, t); err != nil { @@ -1245,15 +1226,15 @@ func (bc *Blockchain) references(ins []transaction.Input) ([]transaction.InOut, for _, inputs := range transaction.GroupInputsByPrevHash(ins) { prevHash := inputs[0].PrevHash - tx, _, err := bc.dao.GetTransaction(prevHash) + unspent, err := bc.dao.GetUnspentCoinState(prevHash) if err != nil { return nil, errors.New("bad input reference") } for _, in := range inputs { - if int(in.PrevIndex) > len(tx.Outputs)-1 { + if int(in.PrevIndex) > len(unspent.States)-1 { return nil, errors.New("bad input reference") } - references = append(references, transaction.InOut{In: *in, Out: tx.Outputs[in.PrevIndex]}) + references = append(references, transaction.InOut{In: *in, Out: unspent.States[in.PrevIndex].Output}) } } return references, nil @@ -1430,17 +1411,26 @@ func (bc *Blockchain) calculateBonus(claims []transaction.Input) (util.Fixed8, e for _, group := range inputs { h := group[0].PrevHash - claimable, err := bc.getUnclaimed(h) - if err != nil || len(claimable) == 0 { - return 0, errors.New("no unclaimed inputs") + unspent, err := bc.dao.GetUnspentCoinState(h) + if err != nil { + return 0, err } for _, c := range group { - s, ok := claimable[c.PrevIndex] - if !ok { + if len(unspent.States) <= int(c.PrevIndex) { return 0, fmt.Errorf("can't find spent coins for %s (%d)", c.PrevHash.StringLE(), c.PrevIndex) } - unclaimed = append(unclaimed, s) + if unspent.States[c.PrevIndex].State&state.CoinSpent == 0 { + return 0, fmt.Errorf("not spent yet: %s/%d", c.PrevHash.StringLE(), c.PrevIndex) + } + if unspent.States[c.PrevIndex].State&state.CoinClaimed != 0 { + return 0, fmt.Errorf("already claimed: %s/%d", c.PrevHash.StringLE(), c.PrevIndex) + } + unclaimed = append(unclaimed, &spentCoin{ + Output: &unspent.States[c.PrevIndex].Output, + StartHeight: unspent.Height, + EndHeight: unspent.States[c.PrevIndex].SpendHeight, + }) } } @@ -1460,29 +1450,6 @@ func (bc *Blockchain) calculateBonusInternal(scs []*spentCoin) (util.Fixed8, err return claimed, nil } -func (bc *Blockchain) getUnclaimed(h util.Uint256) (map[uint16]*spentCoin, error) { - tx, txHeight, err := bc.GetTransaction(h) - if err != nil { - return nil, err - } - - scs, err := bc.dao.GetSpentCoinState(h) - if err != nil { - return nil, err - } - - result := make(map[uint16]*spentCoin) - for i, height := range scs.Items { - result[i] = &spentCoin{ - Output: &tx.Outputs[i], - StartHeight: txHeight, - EndHeight: height, - } - } - - return result, nil -} - // isTxStillRelevant is a callback for mempool transaction filtering after the // new block addition. It returns false for transactions already present in the // chain (added by the new block), transactions using some inputs that are @@ -1714,20 +1681,20 @@ func (bc *Blockchain) GetValidators(txes ...*transaction.Transaction) ([]*keys.P } for hash, inputs := range group { - prevTx, _, err := cache.GetTransaction(hash) + unspent, err := cache.GetUnspentCoinState(hash) if err != nil { return nil, err } // process inputs for _, input := range inputs { - prevOutput := prevTx.Outputs[input.PrevIndex] + prevOutput := &unspent.States[input.PrevIndex].Output accountState, err := cache.GetAccountStateOrNew(prevOutput.ScriptHash) if err != nil { return nil, err } // process account state votes: if there are any -> validators will be updated. - if err = processTXWithValidatorsSubtract(&prevOutput, accountState, cache); err != nil { + if err = processTXWithValidatorsSubtract(prevOutput, accountState, cache); err != nil { return nil, err } delete(accountState.Balances, prevOutput.AssetID) diff --git a/pkg/core/dao.go b/pkg/core/dao.go index f211a2d42..4a2564797 100644 --- a/pkg/core/dao.go +++ b/pkg/core/dao.go @@ -175,20 +175,6 @@ func (dao *dao) AppendNEP5Transfer(acc util.Uint160, tr *state.NEP5Transfer) err // -- start unspent coins. -// GetUnspentCoinStateOrNew gets UnspentCoinState from temporary or persistent Store -// and return it. If it's not present in both stores, returns a new -// UnspentCoinState. -func (dao *dao) GetUnspentCoinStateOrNew(hash util.Uint256) (*state.UnspentCoin, error) { - unspent, err := dao.GetUnspentCoinState(hash) - if err != nil { - if err != storage.ErrKeyNotFound { - return nil, err - } - unspent = state.NewUnspentCoin(0) - } - return unspent, nil -} - // GetUnspentCoinState retrieves UnspentCoinState from the given store. func (dao *dao) GetUnspentCoinState(hash util.Uint256) (*state.UnspentCoin, error) { unspent := &state.UnspentCoin{} @@ -208,45 +194,6 @@ func (dao *dao) PutUnspentCoinState(hash util.Uint256, ucs *state.UnspentCoin) e // -- end unspent coins. -// -- start spent coins. - -// GetSpentCoinsOrNew returns spent coins from store. -func (dao *dao) GetSpentCoinsOrNew(hash util.Uint256, height uint32) (*state.SpentCoin, error) { - spent, err := dao.GetSpentCoinState(hash) - if err != nil { - if err != storage.ErrKeyNotFound { - return nil, err - } - spent = state.NewSpentCoin(height) - } - return spent, nil -} - -// GetSpentCoinState gets SpentCoinState from the given store. -func (dao *dao) GetSpentCoinState(hash util.Uint256) (*state.SpentCoin, error) { - spent := &state.SpentCoin{} - key := storage.AppendPrefix(storage.STSpentCoin, hash.BytesLE()) - err := dao.GetAndDecode(spent, key) - if err != nil { - return nil, err - } - return spent, nil -} - -// PutSpentCoinState puts given SpentCoinState into the given store. -func (dao *dao) PutSpentCoinState(hash util.Uint256, scs *state.SpentCoin) error { - key := storage.AppendPrefix(storage.STSpentCoin, hash.BytesLE()) - return dao.Put(scs, key) -} - -// DeleteSpentCoinState deletes given SpentCoinState from the given store. -func (dao *dao) DeleteSpentCoinState(hash util.Uint256) error { - key := storage.AppendPrefix(storage.STSpentCoin, hash.BytesLE()) - return dao.store.Delete(key) -} - -// -- end spent coins. - // -- start validator. // GetValidatorStateOrNew gets validator from store or created new one in case of error. @@ -590,7 +537,7 @@ func (dao *dao) IsDoubleSpend(tx *transaction.Transaction) bool { return false } for _, input := range inputs { - if int(input.PrevIndex) >= len(unspent.States) || (unspent.States[input.PrevIndex]&state.CoinSpent) != 0 { + if int(input.PrevIndex) >= len(unspent.States) || (unspent.States[input.PrevIndex].State&state.CoinSpent) != 0 { return true } } @@ -605,13 +552,12 @@ func (dao *dao) IsDoubleClaim(claim *transaction.ClaimTX) bool { } for _, inputs := range transaction.GroupInputsByPrevHash(claim.Claims) { prevHash := inputs[0].PrevHash - scs, err := dao.GetSpentCoinState(prevHash) + unspent, err := dao.GetUnspentCoinState(prevHash) if err != nil { return true } for _, input := range inputs { - _, ok := scs.Items[input.PrevIndex] - if !ok { + if int(input.PrevIndex) >= len(unspent.States) || (unspent.States[input.PrevIndex].State&state.CoinClaimed) != 0 { return true } } diff --git a/pkg/core/dao_test.go b/pkg/core/dao_test.go index 962d16df0..94125ecd7 100644 --- a/pkg/core/dao_test.go +++ b/pkg/core/dao_test.go @@ -94,14 +94,6 @@ func TestDeleteContractState(t *testing.T) { require.Nil(t, gotContractState) } -func TestGetUnspentCoinStateOrNew_New(t *testing.T) { - dao := newDao(storage.NewMemoryStore()) - hash := random.Uint256() - unspentCoinState, err := dao.GetUnspentCoinStateOrNew(hash) - require.NoError(t, err) - require.NotNil(t, unspentCoinState) -} - func TestGetUnspentCoinState_Err(t *testing.T) { dao := newDao(storage.NewMemoryStore()) hash := random.Uint256() @@ -113,7 +105,7 @@ func TestGetUnspentCoinState_Err(t *testing.T) { func TestPutGetUnspentCoinState(t *testing.T) { dao := newDao(storage.NewMemoryStore()) hash := random.Uint256() - unspentCoinState := &state.UnspentCoin{States: []state.Coin{}} + unspentCoinState := &state.UnspentCoin{Height: 42, States: []state.OutputState{}} err := dao.PutUnspentCoinState(hash, unspentCoinState) require.NoError(t, err) gotUnspentCoinState, err := dao.GetUnspentCoinState(hash) @@ -121,46 +113,6 @@ func TestPutGetUnspentCoinState(t *testing.T) { require.Equal(t, unspentCoinState, gotUnspentCoinState) } -func TestGetSpentCoinStateOrNew_New(t *testing.T) { - dao := newDao(storage.NewMemoryStore()) - hash := random.Uint256() - spentCoinState, err := dao.GetSpentCoinsOrNew(hash, 1) - require.NoError(t, err) - require.NotNil(t, spentCoinState) -} - -func TestPutAndGetSpentCoinState(t *testing.T) { - dao := newDao(storage.NewMemoryStore()) - hash := random.Uint256() - spentCoinState := &state.SpentCoin{Items: make(map[uint16]uint32)} - err := dao.PutSpentCoinState(hash, spentCoinState) - require.NoError(t, err) - gotSpentCoinState, err := dao.GetSpentCoinState(hash) - require.NoError(t, err) - require.Equal(t, spentCoinState, gotSpentCoinState) -} - -func TestGetSpentCoinState_Err(t *testing.T) { - dao := newDao(storage.NewMemoryStore()) - hash := random.Uint256() - spentCoinState, err := dao.GetSpentCoinState(hash) - require.Error(t, err) - require.Nil(t, spentCoinState) -} - -func TestDeleteSpentCoinState(t *testing.T) { - dao := newDao(storage.NewMemoryStore()) - hash := random.Uint256() - spentCoinState := &state.SpentCoin{Items: make(map[uint16]uint32)} - err := dao.PutSpentCoinState(hash, spentCoinState) - require.NoError(t, err) - err = dao.DeleteSpentCoinState(hash) - require.NoError(t, err) - gotSpentCoinState, err := dao.GetSpentCoinState(hash) - require.Error(t, err) - require.Nil(t, gotSpentCoinState) -} - func TestGetValidatorStateOrNew_New(t *testing.T) { dao := newDao(storage.NewMemoryStore()) publicKey := &keys.PublicKey{} diff --git a/pkg/core/state/spent_coin.go b/pkg/core/state/spent_coin.go deleted file mode 100644 index d1f2c3553..000000000 --- a/pkg/core/state/spent_coin.go +++ /dev/null @@ -1,46 +0,0 @@ -package state - -import "github.com/nspcc-dev/neo-go/pkg/io" - -// SpentCoin represents the state of a spent coin. -type SpentCoin struct { - TxHeight uint32 - - // A mapping between the index of the prevIndex and block height. - Items map[uint16]uint32 -} - -// NewSpentCoin returns a new SpentCoin object. -func NewSpentCoin(height uint32) *SpentCoin { - return &SpentCoin{ - TxHeight: height, - Items: make(map[uint16]uint32), - } -} - -// DecodeBinary implements Serializable interface. -func (s *SpentCoin) DecodeBinary(br *io.BinReader) { - s.TxHeight = br.ReadU32LE() - - s.Items = make(map[uint16]uint32) - lenItems := br.ReadVarUint() - for i := 0; i < int(lenItems); i++ { - var ( - key uint16 - value uint32 - ) - key = br.ReadU16LE() - value = br.ReadU32LE() - s.Items[key] = value - } -} - -// EncodeBinary implements Serializable interface. -func (s *SpentCoin) EncodeBinary(bw *io.BinWriter) { - bw.WriteU32LE(s.TxHeight) - bw.WriteVarUint(uint64(len(s.Items))) - for k, v := range s.Items { - bw.WriteU16LE(k) - bw.WriteU32LE(v) - } -} diff --git a/pkg/core/state/spent_coin_test.go b/pkg/core/state/spent_coin_test.go deleted file mode 100644 index 8759a9ba3..000000000 --- a/pkg/core/state/spent_coin_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package state - -import ( - "testing" - - "github.com/nspcc-dev/neo-go/pkg/io" - "github.com/stretchr/testify/assert" -) - -func TestEncodeDecodeSpentCoin(t *testing.T) { - spent := &SpentCoin{ - TxHeight: 1001, - Items: map[uint16]uint32{ - 1: 3, - 2: 8, - 4: 100, - }, - } - - buf := io.NewBufBinWriter() - spent.EncodeBinary(buf.BinWriter) - assert.Nil(t, buf.Err) - spentDecode := new(SpentCoin) - r := io.NewBinReaderFromBuf(buf.Bytes()) - spentDecode.DecodeBinary(r) - assert.Nil(t, r.Err) - assert.Equal(t, spent, spentDecode) -} diff --git a/pkg/core/state/unspent_coin.go b/pkg/core/state/unspent_coin.go index d153ef639..a0f1e635e 100644 --- a/pkg/core/state/unspent_coin.go +++ b/pkg/core/state/unspent_coin.go @@ -1,38 +1,60 @@ package state import ( + "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/io" ) // UnspentCoin hold the state of a unspent coin. type UnspentCoin struct { - States []Coin + Height uint32 + States []OutputState +} + +// OutputState combines transaction output (UTXO) and its state +// (spent/claimed...) along with the height of spend (if it's spent). +type OutputState struct { + transaction.Output + + SpendHeight uint32 + State Coin } // NewUnspentCoin returns a new unspent coin state with N confirmed states. -func NewUnspentCoin(n int) *UnspentCoin { +func NewUnspentCoin(height uint32, tx *transaction.Transaction) *UnspentCoin { u := &UnspentCoin{ - States: make([]Coin, n), + Height: height, + States: make([]OutputState, len(tx.Outputs)), } - for i := 0; i < n; i++ { - u.States[i] = CoinConfirmed + for i := range tx.Outputs { + u.States[i] = OutputState{Output: tx.Outputs[i]} } return u } // EncodeBinary encodes UnspentCoin to the given BinWriter. func (s *UnspentCoin) EncodeBinary(bw *io.BinWriter) { + bw.WriteU32LE(s.Height) + bw.WriteArray(s.States) bw.WriteVarUint(uint64(len(s.States))) - for _, state := range s.States { - bw.WriteB(byte(state)) - } } // DecodeBinary decodes UnspentCoin from the given BinReader. func (s *UnspentCoin) DecodeBinary(br *io.BinReader) { - lenStates := br.ReadVarUint() - s.States = make([]Coin, lenStates) - for i := 0; i < int(lenStates); i++ { - s.States[i] = Coin(br.ReadB()) - } + s.Height = br.ReadU32LE() + br.ReadArray(&s.States) +} + +// EncodeBinary implements Serializable interface. +func (o *OutputState) EncodeBinary(w *io.BinWriter) { + o.Output.EncodeBinary(w) + w.WriteU32LE(o.SpendHeight) + w.WriteB(byte(o.State)) +} + +// DecodeBinary implements Serializable interface. +func (o *OutputState) DecodeBinary(r *io.BinReader) { + o.Output.DecodeBinary(r) + o.SpendHeight = r.ReadU32LE() + o.State = Coin(r.ReadB()) } diff --git a/pkg/core/state/unspent_coin_test.go b/pkg/core/state/unspent_coin_test.go index 85f82e331..2c9a9274d 100644 --- a/pkg/core/state/unspent_coin_test.go +++ b/pkg/core/state/unspent_coin_test.go @@ -3,18 +3,44 @@ package state import ( "testing" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/internal/random" "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/stretchr/testify/assert" ) func TestDecodeEncodeUnspentCoin(t *testing.T) { unspent := &UnspentCoin{ - States: []Coin{ - CoinConfirmed, - CoinSpent, - CoinSpent, - CoinSpent, - CoinConfirmed, + Height: 100500, + States: []OutputState{ + { + Output: transaction.Output{ + AssetID: random.Uint256(), + Amount: util.Fixed8(42), + ScriptHash: random.Uint160(), + }, + SpendHeight: 201000, + State: CoinSpent, + }, + { + Output: transaction.Output{ + AssetID: random.Uint256(), + Amount: util.Fixed8(420), + ScriptHash: random.Uint160(), + }, + SpendHeight: 0, + State: CoinConfirmed, + }, + { + Output: transaction.Output{ + AssetID: random.Uint256(), + Amount: util.Fixed8(4200), + ScriptHash: random.Uint160(), + }, + SpendHeight: 111000, + State: CoinSpent & CoinClaimed, + }, }, } From 8c902a72237b7969cd32ecec26386c9ab7be5f0c Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Wed, 11 Mar 2020 12:13:02 +0300 Subject: [PATCH 7/7] core: cache UnspentCoins in cachedDao 1.5M block import time (VerifyBlocks disabled) on AMD Ryzen 5 1600/16GB/HDD, before: real 159m16.551s user 69m58.279s sys 7m34.334s after: real 139m41.836s user 67m12.477s sys 6m19.420s 12% which is even a bit more than could be expected from inputs analysis (that has around 10% cache hits for a block-wide cache), worth doing. --- pkg/core/cacheddao.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/pkg/core/cacheddao.go b/pkg/core/cacheddao.go index 65e61c362..8e8d03e7d 100644 --- a/pkg/core/cacheddao.go +++ b/pkg/core/cacheddao.go @@ -13,13 +13,15 @@ type cachedDao struct { dao accounts map[util.Uint160]*state.Account contracts map[util.Uint160]*state.Contract + unspents map[util.Uint256]*state.UnspentCoin } // newCachedDao returns new cachedDao wrapping around given backing store. func newCachedDao(backend storage.Store) *cachedDao { accs := make(map[util.Uint160]*state.Account) ctrs := make(map[util.Uint160]*state.Contract) - return &cachedDao{*newDao(backend), accs, ctrs} + unspents := make(map[util.Uint256]*state.UnspentCoin) + return &cachedDao{*newDao(backend), accs, ctrs, unspents} } // GetAccountStateOrNew retrieves Account from cache or underlying Store @@ -69,6 +71,20 @@ func (cd *cachedDao) DeleteContractState(hash util.Uint160) error { return cd.dao.DeleteContractState(hash) } +// GetUnspentCoinState retrieves UnspentCoin from cache or underlying Store. +func (cd *cachedDao) GetUnspentCoinState(hash util.Uint256) (*state.UnspentCoin, error) { + if cd.unspents[hash] != nil { + return cd.unspents[hash], nil + } + return cd.dao.GetUnspentCoinState(hash) +} + +// PutUnspentCoinState saves given UnspentCoin in the cache. +func (cd *cachedDao) PutUnspentCoinState(hash util.Uint256, ucs *state.UnspentCoin) error { + cd.unspents[hash] = ucs + return nil +} + // Persist flushes all the changes made into the (supposedly) persistent // underlying store. func (cd *cachedDao) Persist() (int, error) { @@ -78,5 +94,11 @@ func (cd *cachedDao) Persist() (int, error) { return 0, err } } + for hash := range cd.unspents { + err := cd.dao.PutUnspentCoinState(hash, cd.unspents[hash]) + if err != nil { + return 0, err + } + } return cd.dao.Persist() }