From 23464401bcdd097d4667eebeb5d45690be41df2c Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 9 Mar 2020 18:56:24 +0300 Subject: [PATCH] core/state: merge spent and unspent coins state, use it to store more things This change reduces pressure on DB by doing the following things: * not storing additional KV pair for SpentCoin * storing Output right in the UnspentCoin, thus eliminating the need to get a full transaction from DB At the same time it makes UnspentCoin more fat and hot, but it should probably worth it. Also drop `GetUnspentCoinStateOrNew` as it shouldn't ever existed, UTXOs can't come out of nowhere. 1.5M block import time (VerifyBlocks disabled) on AMD Ryzen 5 1600/16GB/HDD, before: real 302m9.895s user 96m17.200s sys 13m37.084s after: real 159m16.551s user 69m58.279s sys 7m34.334s So it's almost two-fold which is a great improvement. --- pkg/core/blockchain.go | 125 ++++++++++------------------ pkg/core/dao.go | 60 +------------ pkg/core/dao_test.go | 50 +---------- pkg/core/state/spent_coin.go | 46 ---------- pkg/core/state/spent_coin_test.go | 28 ------- pkg/core/state/unspent_coin.go | 48 ++++++++--- pkg/core/state/unspent_coin_test.go | 38 +++++++-- 7 files changed, 117 insertions(+), 278 deletions(-) delete mode 100644 pkg/core/state/spent_coin.go delete mode 100644 pkg/core/state/spent_coin_test.go diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 23f2610cc..305058a37 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -29,7 +29,7 @@ import ( // Tuning parameters. const ( headerBatchCount = 2000 - version = "0.0.6" + version = "0.0.7" // This one comes from C# code and it's different from the constant used // when creating an asset with Neo.Asset.Create interop call. It looks @@ -475,7 +475,7 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { return err } - if err := cache.PutUnspentCoinState(tx.Hash(), state.NewUnspentCoin(len(tx.Outputs))); err != nil { + if err := cache.PutUnspentCoinState(tx.Hash(), state.NewUnspentCoin(block.Index, tx)); err != nil { return err } @@ -487,22 +487,20 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { // Process TX inputs that are grouped by previous hash. for _, inputs := range transaction.GroupInputsByPrevHash(tx.Inputs) { prevHash := inputs[0].PrevHash - prevTX, prevTXHeight, err := bc.dao.GetTransaction(prevHash) - if err != nil { - return fmt.Errorf("could not find previous TX: %s", prevHash) - } - unspent, err := cache.GetUnspentCoinStateOrNew(prevHash) + unspent, err := cache.GetUnspentCoinState(prevHash) if err != nil { return err } - spentCoin, err := cache.GetSpentCoinsOrNew(prevHash, prevTXHeight) - if err != nil { - return err - } - oldSpentCoinLen := len(spentCoin.Items) for _, input := range inputs { - unspent.States[input.PrevIndex] |= state.CoinSpent - prevTXOutput := prevTX.Outputs[input.PrevIndex] + if len(unspent.States) <= int(input.PrevIndex) { + return fmt.Errorf("bad input: %s/%d", input.PrevHash.StringLE(), input.PrevIndex) + } + if unspent.States[input.PrevIndex].State&state.CoinSpent != 0 { + return fmt.Errorf("double spend: %s/%d", input.PrevHash.StringLE(), input.PrevIndex) + } + unspent.States[input.PrevIndex].State |= state.CoinSpent + unspent.States[input.PrevIndex].SpendHeight = block.Index + prevTXOutput := &unspent.States[input.PrevIndex].Output account, err := cache.GetAccountStateOrNew(prevTXOutput.ScriptHash) if err != nil { return err @@ -510,14 +508,13 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { if prevTXOutput.AssetID.Equals(GoverningTokenID()) { account.Unclaimed = append(account.Unclaimed, state.UnclaimedBalance{ - Tx: prevTX.Hash(), + Tx: input.PrevHash, Index: input.PrevIndex, - Start: prevTXHeight, + Start: unspent.Height, End: block.Index, Value: prevTXOutput.Amount, }) - spentCoin.Items[input.PrevIndex] = block.Index - if err = processTXWithValidatorsSubtract(&prevTXOutput, account, cache); err != nil { + if err = processTXWithValidatorsSubtract(prevTXOutput, account, cache); err != nil { return err } } @@ -548,11 +545,6 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { if err = cache.PutUnspentCoinState(prevHash, unspent); err != nil { return err } - if oldSpentCoinLen != len(spentCoin.Items) { - if err = cache.PutSpentCoinState(prevHash, spentCoin); err != nil { - return err - } - } } // Process the underlying type of the TX. @@ -588,17 +580,18 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { // Remove claimed NEO from spent coins making it unavalaible for // additional claims. for _, input := range t.Claims { - scs, err := cache.GetSpentCoinState(input.PrevHash) + scs, err := cache.GetUnspentCoinState(input.PrevHash) if err == nil { - _, ok := scs.Items[input.PrevIndex] - if !ok { - err = errors.New("no spent coin state") + if len(scs.States) <= int(input.PrevIndex) { + err = errors.New("invalid claim index") + } else if scs.States[input.PrevIndex].State&state.CoinClaimed != 0 { + err = errors.New("double claim") } } if err != nil { // We can't really do anything about it // as it's a transaction in a signed block. - bc.log.Warn("DOUBLE CLAIM", + bc.log.Warn("FALSE OR DOUBLE CLAIM", zap.String("PrevHash", input.PrevHash.StringLE()), zap.Uint16("PrevIndex", input.PrevIndex), zap.String("tx", tx.Hash().StringLE()), @@ -611,14 +604,13 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { break } - prevTx, _, err := cache.GetTransaction(input.PrevHash) + acc, err := cache.GetAccountState(scs.States[input.PrevIndex].ScriptHash) if err != nil { return err - } else if int(input.PrevIndex) > len(prevTx.Outputs) { - return errors.New("invalid input in claim") } - acc, err := cache.GetAccountState(prevTx.Outputs[input.PrevIndex].ScriptHash) - if err != nil { + + scs.States[input.PrevIndex].State |= state.CoinClaimed + if err = cache.PutUnspentCoinState(input.PrevHash, scs); err != nil { return err } @@ -643,17 +635,6 @@ func (bc *Blockchain) storeBlock(block *block.Block) error { } else if err := cache.PutAccountState(acc); err != nil { return err } - - delete(scs.Items, input.PrevIndex) - if len(scs.Items) > 0 { - if err = cache.PutSpentCoinState(input.PrevHash, scs); err != nil { - return err - } - } else { - if err = cache.DeleteSpentCoinState(input.PrevHash); err != nil { - return err - } - } } case *transaction.EnrollmentTX: if err := processEnrollmentTX(cache, t); err != nil { @@ -1245,15 +1226,15 @@ func (bc *Blockchain) references(ins []transaction.Input) ([]transaction.InOut, for _, inputs := range transaction.GroupInputsByPrevHash(ins) { prevHash := inputs[0].PrevHash - tx, _, err := bc.dao.GetTransaction(prevHash) + unspent, err := bc.dao.GetUnspentCoinState(prevHash) if err != nil { return nil, errors.New("bad input reference") } for _, in := range inputs { - if int(in.PrevIndex) > len(tx.Outputs)-1 { + if int(in.PrevIndex) > len(unspent.States)-1 { return nil, errors.New("bad input reference") } - references = append(references, transaction.InOut{In: *in, Out: tx.Outputs[in.PrevIndex]}) + references = append(references, transaction.InOut{In: *in, Out: unspent.States[in.PrevIndex].Output}) } } return references, nil @@ -1430,17 +1411,26 @@ func (bc *Blockchain) calculateBonus(claims []transaction.Input) (util.Fixed8, e for _, group := range inputs { h := group[0].PrevHash - claimable, err := bc.getUnclaimed(h) - if err != nil || len(claimable) == 0 { - return 0, errors.New("no unclaimed inputs") + unspent, err := bc.dao.GetUnspentCoinState(h) + if err != nil { + return 0, err } for _, c := range group { - s, ok := claimable[c.PrevIndex] - if !ok { + if len(unspent.States) <= int(c.PrevIndex) { return 0, fmt.Errorf("can't find spent coins for %s (%d)", c.PrevHash.StringLE(), c.PrevIndex) } - unclaimed = append(unclaimed, s) + if unspent.States[c.PrevIndex].State&state.CoinSpent == 0 { + return 0, fmt.Errorf("not spent yet: %s/%d", c.PrevHash.StringLE(), c.PrevIndex) + } + if unspent.States[c.PrevIndex].State&state.CoinClaimed != 0 { + return 0, fmt.Errorf("already claimed: %s/%d", c.PrevHash.StringLE(), c.PrevIndex) + } + unclaimed = append(unclaimed, &spentCoin{ + Output: &unspent.States[c.PrevIndex].Output, + StartHeight: unspent.Height, + EndHeight: unspent.States[c.PrevIndex].SpendHeight, + }) } } @@ -1460,29 +1450,6 @@ func (bc *Blockchain) calculateBonusInternal(scs []*spentCoin) (util.Fixed8, err return claimed, nil } -func (bc *Blockchain) getUnclaimed(h util.Uint256) (map[uint16]*spentCoin, error) { - tx, txHeight, err := bc.GetTransaction(h) - if err != nil { - return nil, err - } - - scs, err := bc.dao.GetSpentCoinState(h) - if err != nil { - return nil, err - } - - result := make(map[uint16]*spentCoin) - for i, height := range scs.Items { - result[i] = &spentCoin{ - Output: &tx.Outputs[i], - StartHeight: txHeight, - EndHeight: height, - } - } - - return result, nil -} - // isTxStillRelevant is a callback for mempool transaction filtering after the // new block addition. It returns false for transactions already present in the // chain (added by the new block), transactions using some inputs that are @@ -1714,20 +1681,20 @@ func (bc *Blockchain) GetValidators(txes ...*transaction.Transaction) ([]*keys.P } for hash, inputs := range group { - prevTx, _, err := cache.GetTransaction(hash) + unspent, err := cache.GetUnspentCoinState(hash) if err != nil { return nil, err } // process inputs for _, input := range inputs { - prevOutput := prevTx.Outputs[input.PrevIndex] + prevOutput := &unspent.States[input.PrevIndex].Output accountState, err := cache.GetAccountStateOrNew(prevOutput.ScriptHash) if err != nil { return nil, err } // process account state votes: if there are any -> validators will be updated. - if err = processTXWithValidatorsSubtract(&prevOutput, accountState, cache); err != nil { + if err = processTXWithValidatorsSubtract(prevOutput, accountState, cache); err != nil { return nil, err } delete(accountState.Balances, prevOutput.AssetID) diff --git a/pkg/core/dao.go b/pkg/core/dao.go index f211a2d42..4a2564797 100644 --- a/pkg/core/dao.go +++ b/pkg/core/dao.go @@ -175,20 +175,6 @@ func (dao *dao) AppendNEP5Transfer(acc util.Uint160, tr *state.NEP5Transfer) err // -- start unspent coins. -// GetUnspentCoinStateOrNew gets UnspentCoinState from temporary or persistent Store -// and return it. If it's not present in both stores, returns a new -// UnspentCoinState. -func (dao *dao) GetUnspentCoinStateOrNew(hash util.Uint256) (*state.UnspentCoin, error) { - unspent, err := dao.GetUnspentCoinState(hash) - if err != nil { - if err != storage.ErrKeyNotFound { - return nil, err - } - unspent = state.NewUnspentCoin(0) - } - return unspent, nil -} - // GetUnspentCoinState retrieves UnspentCoinState from the given store. func (dao *dao) GetUnspentCoinState(hash util.Uint256) (*state.UnspentCoin, error) { unspent := &state.UnspentCoin{} @@ -208,45 +194,6 @@ func (dao *dao) PutUnspentCoinState(hash util.Uint256, ucs *state.UnspentCoin) e // -- end unspent coins. -// -- start spent coins. - -// GetSpentCoinsOrNew returns spent coins from store. -func (dao *dao) GetSpentCoinsOrNew(hash util.Uint256, height uint32) (*state.SpentCoin, error) { - spent, err := dao.GetSpentCoinState(hash) - if err != nil { - if err != storage.ErrKeyNotFound { - return nil, err - } - spent = state.NewSpentCoin(height) - } - return spent, nil -} - -// GetSpentCoinState gets SpentCoinState from the given store. -func (dao *dao) GetSpentCoinState(hash util.Uint256) (*state.SpentCoin, error) { - spent := &state.SpentCoin{} - key := storage.AppendPrefix(storage.STSpentCoin, hash.BytesLE()) - err := dao.GetAndDecode(spent, key) - if err != nil { - return nil, err - } - return spent, nil -} - -// PutSpentCoinState puts given SpentCoinState into the given store. -func (dao *dao) PutSpentCoinState(hash util.Uint256, scs *state.SpentCoin) error { - key := storage.AppendPrefix(storage.STSpentCoin, hash.BytesLE()) - return dao.Put(scs, key) -} - -// DeleteSpentCoinState deletes given SpentCoinState from the given store. -func (dao *dao) DeleteSpentCoinState(hash util.Uint256) error { - key := storage.AppendPrefix(storage.STSpentCoin, hash.BytesLE()) - return dao.store.Delete(key) -} - -// -- end spent coins. - // -- start validator. // GetValidatorStateOrNew gets validator from store or created new one in case of error. @@ -590,7 +537,7 @@ func (dao *dao) IsDoubleSpend(tx *transaction.Transaction) bool { return false } for _, input := range inputs { - if int(input.PrevIndex) >= len(unspent.States) || (unspent.States[input.PrevIndex]&state.CoinSpent) != 0 { + if int(input.PrevIndex) >= len(unspent.States) || (unspent.States[input.PrevIndex].State&state.CoinSpent) != 0 { return true } } @@ -605,13 +552,12 @@ func (dao *dao) IsDoubleClaim(claim *transaction.ClaimTX) bool { } for _, inputs := range transaction.GroupInputsByPrevHash(claim.Claims) { prevHash := inputs[0].PrevHash - scs, err := dao.GetSpentCoinState(prevHash) + unspent, err := dao.GetUnspentCoinState(prevHash) if err != nil { return true } for _, input := range inputs { - _, ok := scs.Items[input.PrevIndex] - if !ok { + if int(input.PrevIndex) >= len(unspent.States) || (unspent.States[input.PrevIndex].State&state.CoinClaimed) != 0 { return true } } diff --git a/pkg/core/dao_test.go b/pkg/core/dao_test.go index 962d16df0..94125ecd7 100644 --- a/pkg/core/dao_test.go +++ b/pkg/core/dao_test.go @@ -94,14 +94,6 @@ func TestDeleteContractState(t *testing.T) { require.Nil(t, gotContractState) } -func TestGetUnspentCoinStateOrNew_New(t *testing.T) { - dao := newDao(storage.NewMemoryStore()) - hash := random.Uint256() - unspentCoinState, err := dao.GetUnspentCoinStateOrNew(hash) - require.NoError(t, err) - require.NotNil(t, unspentCoinState) -} - func TestGetUnspentCoinState_Err(t *testing.T) { dao := newDao(storage.NewMemoryStore()) hash := random.Uint256() @@ -113,7 +105,7 @@ func TestGetUnspentCoinState_Err(t *testing.T) { func TestPutGetUnspentCoinState(t *testing.T) { dao := newDao(storage.NewMemoryStore()) hash := random.Uint256() - unspentCoinState := &state.UnspentCoin{States: []state.Coin{}} + unspentCoinState := &state.UnspentCoin{Height: 42, States: []state.OutputState{}} err := dao.PutUnspentCoinState(hash, unspentCoinState) require.NoError(t, err) gotUnspentCoinState, err := dao.GetUnspentCoinState(hash) @@ -121,46 +113,6 @@ func TestPutGetUnspentCoinState(t *testing.T) { require.Equal(t, unspentCoinState, gotUnspentCoinState) } -func TestGetSpentCoinStateOrNew_New(t *testing.T) { - dao := newDao(storage.NewMemoryStore()) - hash := random.Uint256() - spentCoinState, err := dao.GetSpentCoinsOrNew(hash, 1) - require.NoError(t, err) - require.NotNil(t, spentCoinState) -} - -func TestPutAndGetSpentCoinState(t *testing.T) { - dao := newDao(storage.NewMemoryStore()) - hash := random.Uint256() - spentCoinState := &state.SpentCoin{Items: make(map[uint16]uint32)} - err := dao.PutSpentCoinState(hash, spentCoinState) - require.NoError(t, err) - gotSpentCoinState, err := dao.GetSpentCoinState(hash) - require.NoError(t, err) - require.Equal(t, spentCoinState, gotSpentCoinState) -} - -func TestGetSpentCoinState_Err(t *testing.T) { - dao := newDao(storage.NewMemoryStore()) - hash := random.Uint256() - spentCoinState, err := dao.GetSpentCoinState(hash) - require.Error(t, err) - require.Nil(t, spentCoinState) -} - -func TestDeleteSpentCoinState(t *testing.T) { - dao := newDao(storage.NewMemoryStore()) - hash := random.Uint256() - spentCoinState := &state.SpentCoin{Items: make(map[uint16]uint32)} - err := dao.PutSpentCoinState(hash, spentCoinState) - require.NoError(t, err) - err = dao.DeleteSpentCoinState(hash) - require.NoError(t, err) - gotSpentCoinState, err := dao.GetSpentCoinState(hash) - require.Error(t, err) - require.Nil(t, gotSpentCoinState) -} - func TestGetValidatorStateOrNew_New(t *testing.T) { dao := newDao(storage.NewMemoryStore()) publicKey := &keys.PublicKey{} diff --git a/pkg/core/state/spent_coin.go b/pkg/core/state/spent_coin.go deleted file mode 100644 index d1f2c3553..000000000 --- a/pkg/core/state/spent_coin.go +++ /dev/null @@ -1,46 +0,0 @@ -package state - -import "github.com/nspcc-dev/neo-go/pkg/io" - -// SpentCoin represents the state of a spent coin. -type SpentCoin struct { - TxHeight uint32 - - // A mapping between the index of the prevIndex and block height. - Items map[uint16]uint32 -} - -// NewSpentCoin returns a new SpentCoin object. -func NewSpentCoin(height uint32) *SpentCoin { - return &SpentCoin{ - TxHeight: height, - Items: make(map[uint16]uint32), - } -} - -// DecodeBinary implements Serializable interface. -func (s *SpentCoin) DecodeBinary(br *io.BinReader) { - s.TxHeight = br.ReadU32LE() - - s.Items = make(map[uint16]uint32) - lenItems := br.ReadVarUint() - for i := 0; i < int(lenItems); i++ { - var ( - key uint16 - value uint32 - ) - key = br.ReadU16LE() - value = br.ReadU32LE() - s.Items[key] = value - } -} - -// EncodeBinary implements Serializable interface. -func (s *SpentCoin) EncodeBinary(bw *io.BinWriter) { - bw.WriteU32LE(s.TxHeight) - bw.WriteVarUint(uint64(len(s.Items))) - for k, v := range s.Items { - bw.WriteU16LE(k) - bw.WriteU32LE(v) - } -} diff --git a/pkg/core/state/spent_coin_test.go b/pkg/core/state/spent_coin_test.go deleted file mode 100644 index 8759a9ba3..000000000 --- a/pkg/core/state/spent_coin_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package state - -import ( - "testing" - - "github.com/nspcc-dev/neo-go/pkg/io" - "github.com/stretchr/testify/assert" -) - -func TestEncodeDecodeSpentCoin(t *testing.T) { - spent := &SpentCoin{ - TxHeight: 1001, - Items: map[uint16]uint32{ - 1: 3, - 2: 8, - 4: 100, - }, - } - - buf := io.NewBufBinWriter() - spent.EncodeBinary(buf.BinWriter) - assert.Nil(t, buf.Err) - spentDecode := new(SpentCoin) - r := io.NewBinReaderFromBuf(buf.Bytes()) - spentDecode.DecodeBinary(r) - assert.Nil(t, r.Err) - assert.Equal(t, spent, spentDecode) -} diff --git a/pkg/core/state/unspent_coin.go b/pkg/core/state/unspent_coin.go index d153ef639..a0f1e635e 100644 --- a/pkg/core/state/unspent_coin.go +++ b/pkg/core/state/unspent_coin.go @@ -1,38 +1,60 @@ package state import ( + "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/io" ) // UnspentCoin hold the state of a unspent coin. type UnspentCoin struct { - States []Coin + Height uint32 + States []OutputState +} + +// OutputState combines transaction output (UTXO) and its state +// (spent/claimed...) along with the height of spend (if it's spent). +type OutputState struct { + transaction.Output + + SpendHeight uint32 + State Coin } // NewUnspentCoin returns a new unspent coin state with N confirmed states. -func NewUnspentCoin(n int) *UnspentCoin { +func NewUnspentCoin(height uint32, tx *transaction.Transaction) *UnspentCoin { u := &UnspentCoin{ - States: make([]Coin, n), + Height: height, + States: make([]OutputState, len(tx.Outputs)), } - for i := 0; i < n; i++ { - u.States[i] = CoinConfirmed + for i := range tx.Outputs { + u.States[i] = OutputState{Output: tx.Outputs[i]} } return u } // EncodeBinary encodes UnspentCoin to the given BinWriter. func (s *UnspentCoin) EncodeBinary(bw *io.BinWriter) { + bw.WriteU32LE(s.Height) + bw.WriteArray(s.States) bw.WriteVarUint(uint64(len(s.States))) - for _, state := range s.States { - bw.WriteB(byte(state)) - } } // DecodeBinary decodes UnspentCoin from the given BinReader. func (s *UnspentCoin) DecodeBinary(br *io.BinReader) { - lenStates := br.ReadVarUint() - s.States = make([]Coin, lenStates) - for i := 0; i < int(lenStates); i++ { - s.States[i] = Coin(br.ReadB()) - } + s.Height = br.ReadU32LE() + br.ReadArray(&s.States) +} + +// EncodeBinary implements Serializable interface. +func (o *OutputState) EncodeBinary(w *io.BinWriter) { + o.Output.EncodeBinary(w) + w.WriteU32LE(o.SpendHeight) + w.WriteB(byte(o.State)) +} + +// DecodeBinary implements Serializable interface. +func (o *OutputState) DecodeBinary(r *io.BinReader) { + o.Output.DecodeBinary(r) + o.SpendHeight = r.ReadU32LE() + o.State = Coin(r.ReadB()) } diff --git a/pkg/core/state/unspent_coin_test.go b/pkg/core/state/unspent_coin_test.go index 85f82e331..2c9a9274d 100644 --- a/pkg/core/state/unspent_coin_test.go +++ b/pkg/core/state/unspent_coin_test.go @@ -3,18 +3,44 @@ package state import ( "testing" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/internal/random" "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/stretchr/testify/assert" ) func TestDecodeEncodeUnspentCoin(t *testing.T) { unspent := &UnspentCoin{ - States: []Coin{ - CoinConfirmed, - CoinSpent, - CoinSpent, - CoinSpent, - CoinConfirmed, + Height: 100500, + States: []OutputState{ + { + Output: transaction.Output{ + AssetID: random.Uint256(), + Amount: util.Fixed8(42), + ScriptHash: random.Uint160(), + }, + SpendHeight: 201000, + State: CoinSpent, + }, + { + Output: transaction.Output{ + AssetID: random.Uint256(), + Amount: util.Fixed8(420), + ScriptHash: random.Uint160(), + }, + SpendHeight: 0, + State: CoinConfirmed, + }, + { + Output: transaction.Output{ + AssetID: random.Uint256(), + Amount: util.Fixed8(4200), + ScriptHash: random.Uint160(), + }, + SpendHeight: 111000, + State: CoinSpent & CoinClaimed, + }, }, }