TrueCloudLab/neo-go
8
0
Fork 0
mirror of https://github.com/nspcc-dev/neo-go.git synced 2024-11-27 03:58:06 +00:00
Code Issues Projects Releases Packages Wiki Activity

core/state: merge spent and unspent coins state, use it to store more things

Browse source 
This change reduces pressure on DB by doing the following things:
 * not storing additional KV pair for SpentCoin
 * storing Output right in the UnspentCoin, thus eliminating the need to get a
   full transaction from DB

At the same time it makes UnspentCoin more fat and hot, but it should probably
worth it.

Also drop `GetUnspentCoinStateOrNew` as it shouldn't ever existed, UTXOs
can't come out of nowhere.

1.5M block import time (VerifyBlocks disabled) on AMD Ryzen 5 1600/16GB/HDD,
before:
real    302m9.895s
user    96m17.200s
sys     13m37.084s

after:
real    159m16.551s
user    69m58.279s
sys     7m34.334s

So it's almost two-fold which is a great improvement.
This commit is contained in:
Roman Khimov 2020-03-09 18:56:24 +03:00
parent e1f194ea7b
commit 23464401bc
7 changed files with 117 additions and 278 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

125
pkg/core/blockchain.go
View file

@ -29,7 +29,7 @@ import (
// Tuning parameters.
const (
headerBatchCount = 2000
version = "0.0.6"
version = "0.0.7"
// This one comes from C# code and it's different from the constant used
// when creating an asset with Neo.Asset.Create interop call. It looks
@ -475,7 +475,7 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
return err
}
if err := cache.PutUnspentCoinState(tx.Hash(), state.NewUnspentCoin(len(tx.Outputs))); err != nil {
if err := cache.PutUnspentCoinState(tx.Hash(), state.NewUnspentCoin(block.Index, tx)); err != nil {
return err
}
@ -487,22 +487,20 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
// Process TX inputs that are grouped by previous hash.
for _, inputs := range transaction.GroupInputsByPrevHash(tx.Inputs) {
prevHash := inputs[0].PrevHash
prevTX, prevTXHeight, err := bc.dao.GetTransaction(prevHash)
if err != nil {
return fmt.Errorf("could not find previous TX: %s", prevHash)
}
unspent, err := cache.GetUnspentCoinStateOrNew(prevHash)
unspent, err := cache.GetUnspentCoinState(prevHash)
if err != nil {
return err
}
spentCoin, err := cache.GetSpentCoinsOrNew(prevHash, prevTXHeight)
if err != nil {
return err
}
oldSpentCoinLen := len(spentCoin.Items)
for _, input := range inputs {
unspent.States[input.PrevIndex] |= state.CoinSpent
prevTXOutput := prevTX.Outputs[input.PrevIndex]
if len(unspent.States) <= int(input.PrevIndex) {
return fmt.Errorf("bad input: %s/%d", input.PrevHash.StringLE(), input.PrevIndex)
}
if unspent.States[input.PrevIndex].State&state.CoinSpent != 0 {
return fmt.Errorf("double spend: %s/%d", input.PrevHash.StringLE(), input.PrevIndex)
}
unspent.States[input.PrevIndex].State |= state.CoinSpent
unspent.States[input.PrevIndex].SpendHeight = block.Index
prevTXOutput := &unspent.States[input.PrevIndex].Output
account, err := cache.GetAccountStateOrNew(prevTXOutput.ScriptHash)
if err != nil {
return err
@ -510,14 +508,13 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
if prevTXOutput.AssetID.Equals(GoverningTokenID()) {
account.Unclaimed = append(account.Unclaimed, state.UnclaimedBalance{
Tx: prevTX.Hash(),
Tx: input.PrevHash,
Index: input.PrevIndex,
Start: prevTXHeight,
Start: unspent.Height,
End: block.Index,
Value: prevTXOutput.Amount,
})
spentCoin.Items[input.PrevIndex] = block.Index
if err = processTXWithValidatorsSubtract(&prevTXOutput, account, cache); err != nil {
if err = processTXWithValidatorsSubtract(prevTXOutput, account, cache); err != nil {
return err
}
}
@ -548,11 +545,6 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
if err = cache.PutUnspentCoinState(prevHash, unspent); err != nil {
return err
}
if oldSpentCoinLen != len(spentCoin.Items) {
if err = cache.PutSpentCoinState(prevHash, spentCoin); err != nil {
return err
}
}
}
// Process the underlying type of the TX.
@ -588,17 +580,18 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
// Remove claimed NEO from spent coins making it unavalaible for
// additional claims.
for _, input := range t.Claims {
scs, err := cache.GetSpentCoinState(input.PrevHash)
scs, err := cache.GetUnspentCoinState(input.PrevHash)
if err == nil {
_, ok := scs.Items[input.PrevIndex]
if !ok {
err = errors.New("no spent coin state")
if len(scs.States) <= int(input.PrevIndex) {
err = errors.New("invalid claim index")
} else if scs.States[input.PrevIndex].State&state.CoinClaimed != 0 {
err = errors.New("double claim")
}
}
if err != nil {
// We can't really do anything about it
// as it's a transaction in a signed block.
bc.log.Warn("DOUBLE CLAIM",
bc.log.Warn("FALSE OR DOUBLE CLAIM",
zap.String("PrevHash", input.PrevHash.StringLE()),
zap.Uint16("PrevIndex", input.PrevIndex),
zap.String("tx", tx.Hash().StringLE()),
@ -611,14 +604,13 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
break
}
prevTx, _, err := cache.GetTransaction(input.PrevHash)
acc, err := cache.GetAccountState(scs.States[input.PrevIndex].ScriptHash)
if err != nil {
return err
} else if int(input.PrevIndex) > len(prevTx.Outputs) {
return errors.New("invalid input in claim")
}
acc, err := cache.GetAccountState(prevTx.Outputs[input.PrevIndex].ScriptHash)
if err != nil {
scs.States[input.PrevIndex].State |= state.CoinClaimed
if err = cache.PutUnspentCoinState(input.PrevHash, scs); err != nil {
return err
}
@ -643,17 +635,6 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
} else if err := cache.PutAccountState(acc); err != nil {
return err
}
delete(scs.Items, input.PrevIndex)
if len(scs.Items) > 0 {
if err = cache.PutSpentCoinState(input.PrevHash, scs); err != nil {
return err
}
} else {
if err = cache.DeleteSpentCoinState(input.PrevHash); err != nil {
return err
}
}
}
case *transaction.EnrollmentTX:
if err := processEnrollmentTX(cache, t); err != nil {
@ -1245,15 +1226,15 @@ func (bc *Blockchain) references(ins []transaction.Input) ([]transaction.InOut,
for _, inputs := range transaction.GroupInputsByPrevHash(ins) {
prevHash := inputs[0].PrevHash
tx, _, err := bc.dao.GetTransaction(prevHash)
unspent, err := bc.dao.GetUnspentCoinState(prevHash)
if err != nil {
return nil, errors.New("bad input reference")
}
for _, in := range inputs {
if int(in.PrevIndex) > len(tx.Outputs)-1 {
if int(in.PrevIndex) > len(unspent.States)-1 {
return nil, errors.New("bad input reference")
}
references = append(references, transaction.InOut{In: *in, Out: tx.Outputs[in.PrevIndex]})
references = append(references, transaction.InOut{In: *in, Out: unspent.States[in.PrevIndex].Output})
}
}
return references, nil
@ -1430,17 +1411,26 @@ func (bc *Blockchain) calculateBonus(claims []transaction.Input) (util.Fixed8, e
for _, group := range inputs {
h := group[0].PrevHash
claimable, err := bc.getUnclaimed(h)
if err != nil || len(claimable) == 0 {
return 0, errors.New("no unclaimed inputs")
unspent, err := bc.dao.GetUnspentCoinState(h)
if err != nil {
return 0, err
}
for _, c := range group {
s, ok := claimable[c.PrevIndex]
if !ok {
if len(unspent.States) <= int(c.PrevIndex) {
return 0, fmt.Errorf("can't find spent coins for %s (%d)", c.PrevHash.StringLE(), c.PrevIndex)
}
unclaimed = append(unclaimed, s)
if unspent.States[c.PrevIndex].State&state.CoinSpent == 0 {
return 0, fmt.Errorf("not spent yet: %s/%d", c.PrevHash.StringLE(), c.PrevIndex)
}
if unspent.States[c.PrevIndex].State&state.CoinClaimed != 0 {
return 0, fmt.Errorf("already claimed: %s/%d", c.PrevHash.StringLE(), c.PrevIndex)
}
unclaimed = append(unclaimed, &spentCoin{
Output: &unspent.States[c.PrevIndex].Output,
StartHeight: unspent.Height,
EndHeight: unspent.States[c.PrevIndex].SpendHeight,
})
}
}
@ -1460,29 +1450,6 @@ func (bc *Blockchain) calculateBonusInternal(scs []*spentCoin) (util.Fixed8, err
return claimed, nil
}
func (bc *Blockchain) getUnclaimed(h util.Uint256) (map[uint16]*spentCoin, error) {
tx, txHeight, err := bc.GetTransaction(h)
if err != nil {
return nil, err
}
scs, err := bc.dao.GetSpentCoinState(h)
if err != nil {
return nil, err
}
result := make(map[uint16]*spentCoin)
for i, height := range scs.Items {
result[i] = &spentCoin{
Output: &tx.Outputs[i],
StartHeight: txHeight,
EndHeight: height,
}
}
return result, nil
}
// isTxStillRelevant is a callback for mempool transaction filtering after the
// new block addition. It returns false for transactions already present in the
// chain (added by the new block), transactions using some inputs that are
@ -1714,20 +1681,20 @@ func (bc *Blockchain) GetValidators(txes ...*transaction.Transaction) ([]*keys.P
}
for hash, inputs := range group {
prevTx, _, err := cache.GetTransaction(hash)
unspent, err := cache.GetUnspentCoinState(hash)
if err != nil {
return nil, err
}
// process inputs
for _, input := range inputs {
prevOutput := prevTx.Outputs[input.PrevIndex]
prevOutput := &unspent.States[input.PrevIndex].Output
accountState, err := cache.GetAccountStateOrNew(prevOutput.ScriptHash)
if err != nil {
return nil, err
}
// process account state votes: if there are any -> validators will be updated.
if err = processTXWithValidatorsSubtract(&prevOutput, accountState, cache); err != nil {
if err = processTXWithValidatorsSubtract(prevOutput, accountState, cache); err != nil {
return nil, err
}
delete(accountState.Balances, prevOutput.AssetID)

60
pkg/core/dao.go
View file

@ -175,20 +175,6 @@ func (dao *dao) AppendNEP5Transfer(acc util.Uint160, tr *state.NEP5Transfer) err
// -- start unspent coins.
// GetUnspentCoinStateOrNew gets UnspentCoinState from temporary or persistent Store
// and return it. If it's not present in both stores, returns a new
// UnspentCoinState.
func (dao *dao) GetUnspentCoinStateOrNew(hash util.Uint256) (*state.UnspentCoin, error) {
unspent, err := dao.GetUnspentCoinState(hash)
if err != nil {
if err != storage.ErrKeyNotFound {
return nil, err
}
unspent = state.NewUnspentCoin(0)
}
return unspent, nil
}
// GetUnspentCoinState retrieves UnspentCoinState from the given store.
func (dao *dao) GetUnspentCoinState(hash util.Uint256) (*state.UnspentCoin, error) {
unspent := &state.UnspentCoin{}
@ -208,45 +194,6 @@ func (dao *dao) PutUnspentCoinState(hash util.Uint256, ucs *state.UnspentCoin) e
// -- end unspent coins.
// -- start spent coins.
// GetSpentCoinsOrNew returns spent coins from store.
func (dao *dao) GetSpentCoinsOrNew(hash util.Uint256, height uint32) (*state.SpentCoin, error) {
spent, err := dao.GetSpentCoinState(hash)
if err != nil {
if err != storage.ErrKeyNotFound {
return nil, err
}
spent = state.NewSpentCoin(height)
}
return spent, nil
}
// GetSpentCoinState gets SpentCoinState from the given store.
func (dao *dao) GetSpentCoinState(hash util.Uint256) (*state.SpentCoin, error) {
spent := &state.SpentCoin{}
key := storage.AppendPrefix(storage.STSpentCoin, hash.BytesLE())
err := dao.GetAndDecode(spent, key)
if err != nil {
return nil, err
}
return spent, nil
}
// PutSpentCoinState puts given SpentCoinState into the given store.
func (dao *dao) PutSpentCoinState(hash util.Uint256, scs *state.SpentCoin) error {
key := storage.AppendPrefix(storage.STSpentCoin, hash.BytesLE())
return dao.Put(scs, key)
}
// DeleteSpentCoinState deletes given SpentCoinState from the given store.
func (dao *dao) DeleteSpentCoinState(hash util.Uint256) error {
key := storage.AppendPrefix(storage.STSpentCoin, hash.BytesLE())
return dao.store.Delete(key)
}
// -- end spent coins.
// -- start validator.
// GetValidatorStateOrNew gets validator from store or created new one in case of error.
@ -590,7 +537,7 @@ func (dao *dao) IsDoubleSpend(tx *transaction.Transaction) bool {
return false
}
for _, input := range inputs {
if int(input.PrevIndex) >= len(unspent.States) || (unspent.States[input.PrevIndex]&state.CoinSpent) != 0 {
if int(input.PrevIndex) >= len(unspent.States) || (unspent.States[input.PrevIndex].State&state.CoinSpent) != 0 {
return true
}
}
@ -605,13 +552,12 @@ func (dao *dao) IsDoubleClaim(claim *transaction.ClaimTX) bool {
}
for _, inputs := range transaction.GroupInputsByPrevHash(claim.Claims) {
prevHash := inputs[0].PrevHash
scs, err := dao.GetSpentCoinState(prevHash)
unspent, err := dao.GetUnspentCoinState(prevHash)
if err != nil {
return true
}
for _, input := range inputs {
_, ok := scs.Items[input.PrevIndex]
if !ok {
if int(input.PrevIndex) >= len(unspent.States) || (unspent.States[input.PrevIndex].State&state.CoinClaimed) != 0 {
return true
}
}

50
pkg/core/dao_test.go
View file

@ -94,14 +94,6 @@ func TestDeleteContractState(t *testing.T) {
require.Nil(t, gotContractState)
}
func TestGetUnspentCoinStateOrNew_New(t *testing.T) {
dao := newDao(storage.NewMemoryStore())
hash := random.Uint256()
unspentCoinState, err := dao.GetUnspentCoinStateOrNew(hash)
require.NoError(t, err)
require.NotNil(t, unspentCoinState)
}
func TestGetUnspentCoinState_Err(t *testing.T) {
dao := newDao(storage.NewMemoryStore())
hash := random.Uint256()
@ -113,7 +105,7 @@ func TestGetUnspentCoinState_Err(t *testing.T) {
func TestPutGetUnspentCoinState(t *testing.T) {
dao := newDao(storage.NewMemoryStore())
hash := random.Uint256()
unspentCoinState := &state.UnspentCoin{States: []state.Coin{}}
unspentCoinState := &state.UnspentCoin{Height: 42, States: []state.OutputState{}}
err := dao.PutUnspentCoinState(hash, unspentCoinState)
require.NoError(t, err)
gotUnspentCoinState, err := dao.GetUnspentCoinState(hash)
@ -121,46 +113,6 @@ func TestPutGetUnspentCoinState(t *testing.T) {
require.Equal(t, unspentCoinState, gotUnspentCoinState)
}
func TestGetSpentCoinStateOrNew_New(t *testing.T) {
dao := newDao(storage.NewMemoryStore())
hash := random.Uint256()
spentCoinState, err := dao.GetSpentCoinsOrNew(hash, 1)
require.NoError(t, err)
require.NotNil(t, spentCoinState)
}
func TestPutAndGetSpentCoinState(t *testing.T) {
dao := newDao(storage.NewMemoryStore())
hash := random.Uint256()
spentCoinState := &state.SpentCoin{Items: make(map[uint16]uint32)}
err := dao.PutSpentCoinState(hash, spentCoinState)
require.NoError(t, err)
gotSpentCoinState, err := dao.GetSpentCoinState(hash)
require.NoError(t, err)
require.Equal(t, spentCoinState, gotSpentCoinState)
}
func TestGetSpentCoinState_Err(t *testing.T) {
dao := newDao(storage.NewMemoryStore())
hash := random.Uint256()
spentCoinState, err := dao.GetSpentCoinState(hash)
require.Error(t, err)
require.Nil(t, spentCoinState)
}
func TestDeleteSpentCoinState(t *testing.T) {
dao := newDao(storage.NewMemoryStore())
hash := random.Uint256()
spentCoinState := &state.SpentCoin{Items: make(map[uint16]uint32)}
err := dao.PutSpentCoinState(hash, spentCoinState)
require.NoError(t, err)
err = dao.DeleteSpentCoinState(hash)
require.NoError(t, err)
gotSpentCoinState, err := dao.GetSpentCoinState(hash)
require.Error(t, err)
require.Nil(t, gotSpentCoinState)
}
func TestGetValidatorStateOrNew_New(t *testing.T) {
dao := newDao(storage.NewMemoryStore())
publicKey := &keys.PublicKey{}

46
pkg/core/state/spent_coin.go
View file

@ -1,46 +0,0 @@
package state
import "github.com/nspcc-dev/neo-go/pkg/io"
// SpentCoin represents the state of a spent coin.
type SpentCoin struct {
TxHeight uint32
// A mapping between the index of the prevIndex and block height.
Items map[uint16]uint32
}
// NewSpentCoin returns a new SpentCoin object.
func NewSpentCoin(height uint32) *SpentCoin {
return &SpentCoin{
TxHeight: height,
Items: make(map[uint16]uint32),
}
}
// DecodeBinary implements Serializable interface.
func (s *SpentCoin) DecodeBinary(br *io.BinReader) {
s.TxHeight = br.ReadU32LE()
s.Items = make(map[uint16]uint32)
lenItems := br.ReadVarUint()
for i := 0; i < int(lenItems); i++ {
var (
key uint16
value uint32
)
key = br.ReadU16LE()
value = br.ReadU32LE()
s.Items[key] = value
}
}
// EncodeBinary implements Serializable interface.
func (s *SpentCoin) EncodeBinary(bw *io.BinWriter) {
bw.WriteU32LE(s.TxHeight)
bw.WriteVarUint(uint64(len(s.Items)))
for k, v := range s.Items {
bw.WriteU16LE(k)
bw.WriteU32LE(v)
}
}

28
pkg/core/state/spent_coin_test.go
View file

@ -1,28 +0,0 @@
package state
import (
"testing"
"github.com/nspcc-dev/neo-go/pkg/io"
"github.com/stretchr/testify/assert"
)
func TestEncodeDecodeSpentCoin(t *testing.T) {
spent := &SpentCoin{
TxHeight: 1001,
Items: map[uint16]uint32{
1: 3,
2: 8,
4: 100,
},
}
buf := io.NewBufBinWriter()
spent.EncodeBinary(buf.BinWriter)
assert.Nil(t, buf.Err)
spentDecode := new(SpentCoin)
r := io.NewBinReaderFromBuf(buf.Bytes())
spentDecode.DecodeBinary(r)
assert.Nil(t, r.Err)
assert.Equal(t, spent, spentDecode)
}

46
pkg/core/state/unspent_coin.go
View file

@ -1,38 +1,60 @@
package state
import (
"github.com/nspcc-dev/neo-go/pkg/core/transaction"
"github.com/nspcc-dev/neo-go/pkg/io"
)
// UnspentCoin hold the state of a unspent coin.
type UnspentCoin struct {
States []Coin
Height uint32
States []OutputState
}
// OutputState combines transaction output (UTXO) and its state
// (spent/claimed...) along with the height of spend (if it's spent).
type OutputState struct {
transaction.Output
SpendHeight uint32
State Coin
}
// NewUnspentCoin returns a new unspent coin state with N confirmed states.
func NewUnspentCoin(n int) *UnspentCoin {
func NewUnspentCoin(height uint32, tx *transaction.Transaction) *UnspentCoin {
u := &UnspentCoin{
States: make([]Coin, n),
Height: height,
States: make([]OutputState, len(tx.Outputs)),
}
for i := 0; i < n; i++ {
u.States[i] = CoinConfirmed
for i := range tx.Outputs {
u.States[i] = OutputState{Output: tx.Outputs[i]}
}
return u
}
// EncodeBinary encodes UnspentCoin to the given BinWriter.
func (s *UnspentCoin) EncodeBinary(bw *io.BinWriter) {
bw.WriteU32LE(s.Height)
bw.WriteArray(s.States)
bw.WriteVarUint(uint64(len(s.States)))
for _, state := range s.States {
bw.WriteB(byte(state))
}
}
// DecodeBinary decodes UnspentCoin from the given BinReader.
func (s *UnspentCoin) DecodeBinary(br *io.BinReader) {
lenStates := br.ReadVarUint()
s.States = make([]Coin, lenStates)
for i := 0; i < int(lenStates); i++ {
s.States[i] = Coin(br.ReadB())
s.Height = br.ReadU32LE()
br.ReadArray(&s.States)
}
// EncodeBinary implements Serializable interface.
func (o *OutputState) EncodeBinary(w *io.BinWriter) {
o.Output.EncodeBinary(w)
w.WriteU32LE(o.SpendHeight)
w.WriteB(byte(o.State))
}
// DecodeBinary implements Serializable interface.
func (o *OutputState) DecodeBinary(r *io.BinReader) {
o.Output.DecodeBinary(r)
o.SpendHeight = r.ReadU32LE()
o.State = Coin(r.ReadB())
}

38
pkg/core/state/unspent_coin_test.go
View file

@ -3,18 +3,44 @@ package state
import (
"testing"
"github.com/nspcc-dev/neo-go/pkg/core/transaction"
"github.com/nspcc-dev/neo-go/pkg/internal/random"
"github.com/nspcc-dev/neo-go/pkg/io"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/stretchr/testify/assert"
)
func TestDecodeEncodeUnspentCoin(t *testing.T) {
unspent := &UnspentCoin{
States: []Coin{
CoinConfirmed,
CoinSpent,
CoinSpent,
CoinSpent,
CoinConfirmed,
Height: 100500,
States: []OutputState{
{
Output: transaction.Output{
AssetID: random.Uint256(),
Amount: util.Fixed8(42),
ScriptHash: random.Uint160(),
},
SpendHeight: 201000,
State: CoinSpent,
},
{
Output: transaction.Output{
AssetID: random.Uint256(),
Amount: util.Fixed8(420),
ScriptHash: random.Uint160(),
},
SpendHeight: 0,
State: CoinConfirmed,
},
{
Output: transaction.Output{
AssetID: random.Uint256(),
Amount: util.Fixed8(4200),
ScriptHash: random.Uint160(),
},
SpendHeight: 111000,
State: CoinSpent & CoinClaimed,
},
},
}