From bf20db09e05fae4d7e3ef142b823bab8df32ce7e Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Fri, 29 Jan 2021 17:33:24 +0300 Subject: [PATCH] stateroot: move state-root related logic to core/stateroot --- internal/fakechain/fakechain.go | 14 +-- pkg/consensus/consensus.go | 6 +- pkg/consensus/consensus_test.go | 2 +- pkg/core/blockchain.go | 129 +++----------------------- pkg/core/blockchain_test.go | 2 +- pkg/core/blockchainer/blockchainer.go | 4 +- pkg/core/blockchainer/state_root.go | 13 +++ pkg/core/dao/dao.go | 82 +--------------- pkg/core/helper_test.go | 2 +- pkg/core/native_management_test.go | 4 +- pkg/core/prometheus.go | 12 --- pkg/core/state/mpt_root.go | 113 +++------------------- pkg/core/state/mpt_root_test.go | 54 ++--------- pkg/core/stateroot/module.go | 98 +++++++++++++++++++ pkg/core/stateroot/prometheus.go | 20 ++++ pkg/core/stateroot/store.go | 56 +++++++++++ pkg/rpc/server/server.go | 8 +- pkg/rpc/server/server_test.go | 6 +- scripts/gendump/main.go | 2 +- 19 files changed, 245 insertions(+), 382 deletions(-) create mode 100644 pkg/core/blockchainer/state_root.go create mode 100644 pkg/core/stateroot/module.go create mode 100644 pkg/core/stateroot/prometheus.go create mode 100644 pkg/core/stateroot/store.go diff --git a/internal/fakechain/fakechain.go b/internal/fakechain/fakechain.go index 9d89e91d2..fae3804a7 100644 --- a/internal/fakechain/fakechain.go +++ b/internal/fakechain/fakechain.go @@ -177,11 +177,6 @@ func (chain *FakeChain) AddBlock(block *block.Block) error { return nil } -// AddStateRoot implements Blockchainer interface. -func (chain *FakeChain) AddStateRoot(r *state.MPTRoot) error { - panic("TODO") -} - // BlockHeight implements Feer interface. func (chain *FakeChain) BlockHeight() uint32 { return atomic.LoadUint32(&chain.Blockheight) @@ -279,13 +274,8 @@ func (chain *FakeChain) GetEnrollments() ([]state.Validator, error) { panic("TODO") } -// GetStateProof implements Blockchainer interface. -func (chain *FakeChain) GetStateProof(util.Uint256, []byte) ([][]byte, error) { - panic("TODO") -} - -// GetStateRoot implements Blockchainer interface. -func (chain *FakeChain) GetStateRoot(height uint32) (*state.MPTRootState, error) { +// GetStateModule implements Blockchainer interface. +func (chain *FakeChain) GetStateModule() blockchainer.StateRoot { panic("TODO") } diff --git a/pkg/consensus/consensus.go b/pkg/consensus/consensus.go index 57de2c3ba..03451d438 100644 --- a/pkg/consensus/consensus.go +++ b/pkg/consensus/consensus.go @@ -236,7 +236,7 @@ func (s *service) newPrepareRequest() payload.PrepareRequest { r := new(prepareRequest) if s.stateRootEnabled { r.stateRootEnabled = true - if sr, err := s.Chain.GetStateRoot(s.dbft.BlockIndex - 1); err == nil { + if sr, err := s.Chain.GetStateModule().GetStateRoot(s.dbft.BlockIndex - 1); err == nil { r.stateRoot = sr.Root } else { panic(err) @@ -483,7 +483,7 @@ func (s *service) verifyRequest(p payload.ConsensusPayload) error { return errInvalidVersion } if s.stateRootEnabled { - sr, err := s.Chain.GetStateRoot(s.dbft.BlockIndex - 1) + sr, err := s.Chain.GetStateModule().GetStateRoot(s.dbft.BlockIndex - 1) if err != nil { return err } else if sr.Root != req.stateRoot { @@ -637,7 +637,7 @@ func (s *service) newBlockFromContext(ctx *dbft.Context) block.Block { block.Block.Timestamp = ctx.Timestamp / nsInMs block.Block.Index = ctx.BlockIndex if s.stateRootEnabled { - sr, err := s.Chain.GetStateRoot(ctx.BlockIndex - 1) + sr, err := s.Chain.GetStateModule().GetStateRoot(ctx.BlockIndex - 1) if err != nil { return nil } diff --git a/pkg/consensus/consensus_test.go b/pkg/consensus/consensus_test.go index fa59e0248..fa65fba1b 100644 --- a/pkg/consensus/consensus_test.go +++ b/pkg/consensus/consensus_test.go @@ -321,7 +321,7 @@ func TestService_PrepareRequest(t *testing.T) { prevHash: prevHash, }) - sr, err := srv.Chain.GetStateRoot(srv.dbft.BlockIndex - 1) + sr, err := srv.Chain.GetStateModule().GetStateRoot(srv.dbft.BlockIndex - 1) require.NoError(t, err) checkRequest(t, nil, &prepareRequest{ stateRootEnabled: true, diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 988b864f4..998992a12 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -19,9 +19,9 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/interop" "github.com/nspcc-dev/neo-go/pkg/core/interop/contract" "github.com/nspcc-dev/neo-go/pkg/core/mempool" - "github.com/nspcc-dev/neo-go/pkg/core/mpt" "github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/state" + "github.com/nspcc-dev/neo-go/pkg/core/stateroot" "github.com/nspcc-dev/neo-go/pkg/core/storage" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto" @@ -134,6 +134,8 @@ type Blockchain struct { extensible atomic.Value + stateRoot *stateroot.Module + // Notification subsystem. events chan bcEvent subCh chan interface{} @@ -193,6 +195,8 @@ func NewBlockchain(s storage.Store, cfg config.ProtocolConfiguration, log *zap.L contracts: *native.NewContracts(cfg.P2PSigExtensions), } + bc.stateRoot = stateroot.NewModule(bc, bc.log, bc.dao.Store) + if err := bc.init(); err != nil { return nil, err } @@ -237,7 +241,7 @@ func (bc *Blockchain) init() error { if err != nil { return err } - if err := bc.dao.InitMPT(0, bc.config.KeepOnlyLatestState); err != nil { + if err := bc.stateRoot.Init(0, bc.config.KeepOnlyLatestState); err != nil { return fmt.Errorf("can't init MPT: %w", err) } return bc.storeBlock(genesisBlock, nil) @@ -257,7 +261,7 @@ func (bc *Blockchain) init() error { } bc.blockHeight = bHeight bc.persistedHeight = bHeight - if err = bc.dao.InitMPT(bHeight, bc.config.KeepOnlyLatestState); err != nil { + if err = bc.stateRoot.Init(bHeight, bc.config.KeepOnlyLatestState); err != nil { return fmt.Errorf("can't init MPT at height %d: %w", bHeight, err) } @@ -479,7 +483,7 @@ func (bc *Blockchain) AddBlock(block *block.Block) error { ErrHdrStateRootSetting, bc.config.StateRootInHeader, block.StateRootEnabled) } if bc.config.StateRootInHeader { - if sr := bc.dao.MPT.StateRoot(); block.PrevStateRoot != sr { + if sr := bc.stateRoot.CurrentLocalStateRoot(); block.PrevStateRoot != sr { return fmt.Errorf("%w: %s != %s", ErrHdrInvalidStateRoot, block.PrevStateRoot.StringLE(), sr.StringLE()) } @@ -606,15 +610,9 @@ func (bc *Blockchain) addHeaders(verify bool, headers ...*block.Header) error { return nil } -// GetStateProof returns proof of having key in the MPT with the specified root. -func (bc *Blockchain) GetStateProof(root util.Uint256, key []byte) ([][]byte, error) { - tr := mpt.NewTrie(mpt.NewHashNode(root), false, storage.NewMemCachedStore(bc.dao.Store)) - return tr.GetProof(key) -} - -// GetStateRoot returns state root for a given height. -func (bc *Blockchain) GetStateRoot(height uint32) (*state.MPTRootState, error) { - return bc.dao.GetStateRoot(height) +// GetStateModule returns state root service instance. +func (bc *Blockchain) GetStateModule() blockchainer.StateRoot { + return bc.stateRoot } // storeBlock performs chain update using the block given, it executes all @@ -718,33 +716,14 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error writeBuf.Reset() d := cache.DAO.(*dao.Simple) - if err := d.UpdateMPT(); err != nil { + b := d.GetMPTBatch() + if err := bc.stateRoot.AddMPTBatch(block.Index, b); err != nil { // Here MPT can be left in a half-applied state. // However if this error occurs, this is a bug somewhere in code // because changes applied are the ones from HALTed transactions. return fmt.Errorf("error while trying to apply MPT changes: %w", err) } - root := d.MPT.StateRoot() - var prevHash util.Uint256 - if block.Index > 0 { - prev, err := bc.dao.GetStateRoot(block.Index - 1) - if err != nil { - return fmt.Errorf("can't get previous state root: %w", err) - } - prevHash = hash.DoubleSha256(prev.GetSignedPart()) - } - err = bc.AddStateRoot(&state.MPTRoot{ - MPTRootBase: state.MPTRootBase{ - Index: block.Index, - PrevHash: prevHash, - Root: root, - }, - }) - if err != nil { - return err - } - if bc.config.SaveStorageBatch { bc.lastBatch = cache.DAO.GetBatch() } @@ -767,13 +746,7 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error bc.lock.Unlock() return err } - bc.dao.MPT.Flush() - // Every persist cycle we also compact our in-memory MPT. - persistedHeight := atomic.LoadUint32(&bc.persistedHeight) - if persistedHeight == block.Index-1 { - // 10 is good and roughly estimated to fit remaining trie into 1M of memory. - bc.dao.MPT.Collapse(10) - } + bc.topBlock.Store(block) atomic.StoreUint32(&bc.blockHeight, block.Index) bc.memPool.RemoveStale(func(tx *transaction.Transaction) bool { return bc.IsTxStillRelevant(tx, txpool, false) }, bc) @@ -1582,79 +1555,6 @@ func (bc *Blockchain) IsTxStillRelevant(t *transaction.Transaction, txpool *memp } -// AddStateRoot add new (possibly unverified) state root to the blockchain. -func (bc *Blockchain) AddStateRoot(r *state.MPTRoot) error { - our, err := bc.GetStateRoot(r.Index) - if err == nil { - if our.Flag == state.Verified { - return bc.updateStateHeight(r.Index) - } else if r.Witness == nil && our.Witness != nil { - r.Witness = our.Witness - } - } - if err := bc.verifyStateRoot(r); err != nil { - return fmt.Errorf("invalid state root: %w", err) - } - if r.Index > bc.BlockHeight() { // just put it into the store for future checks - return bc.dao.PutStateRoot(&state.MPTRootState{ - MPTRoot: *r, - Flag: state.Unverified, - }) - } - - flag := state.Unverified - if r.Witness != nil { - if err := bc.verifyStateRootWitness(r); err != nil { - return fmt.Errorf("can't verify signature: %w", err) - } - flag = state.Verified - } - err = bc.dao.PutStateRoot(&state.MPTRootState{ - MPTRoot: *r, - Flag: flag, - }) - if err != nil { - return err - } - return bc.updateStateHeight(r.Index) -} - -func (bc *Blockchain) updateStateHeight(newHeight uint32) error { - h, err := bc.dao.GetCurrentStateRootHeight() - if err != nil { - return fmt.Errorf("can't get current state root height: %w", err) - } else if newHeight == h+1 { - updateStateHeightMetric(newHeight) - return bc.dao.PutCurrentStateRootHeight(h + 1) - } - return nil -} - -// verifyStateRoot checks if state root is valid. -func (bc *Blockchain) verifyStateRoot(r *state.MPTRoot) error { - if r.Index == 0 { - return nil - } - prev, err := bc.GetStateRoot(r.Index - 1) - if err != nil { - return errors.New("can't get previous state root") - } else if !r.PrevHash.Equals(hash.DoubleSha256(prev.GetSignedPart())) { - return errors.New("previous hash mismatch") - } else if prev.Version != r.Version { - return errors.New("version mismatch") - } - return nil -} - -// verifyStateRootWitness verifies that state root signature is correct. -func (bc *Blockchain) verifyStateRootWitness(r *state.MPTRoot) error { - b, err := bc.GetBlock(bc.GetHeaderHash(int(r.Index))) - if err != nil { - return err - } - return bc.VerifyWitness(b.NextConsensus, r, r.Witness, bc.contracts.Policy.GetMaxVerificationGas(bc.dao)) -} - // VerifyTx verifies whether transaction is bonafide or not relative to the // current blockchain state. Note that this verification is completely isolated // from the main node's mempool. @@ -1731,7 +1631,6 @@ func (bc *Blockchain) GetEnrollments() ([]state.Validator, error) { // GetTestVM returns a VM and a Store setup for a test run of some sort of code. func (bc *Blockchain) GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) *vm.VM { d := bc.dao.GetWrapped().(*dao.Simple) - d.MPT = nil systemInterop := bc.newInteropContext(t, d, b, tx) vm := systemInterop.SpawnVM() vm.SetPriceGetter(systemInterop.GetPrice) diff --git a/pkg/core/blockchain_test.go b/pkg/core/blockchain_test.go index 35cfa850a..5503e2667 100644 --- a/pkg/core/blockchain_test.go +++ b/pkg/core/blockchain_test.go @@ -132,7 +132,7 @@ func TestAddBlockStateRoot(t *testing.T) { c.ProtocolConfiguration.StateRootInHeader = true }) - sr, err := bc.GetStateRoot(bc.BlockHeight()) + sr, err := bc.GetStateModule().GetStateRoot(bc.BlockHeight()) require.NoError(t, err) tx := newNEP17Transfer(bc.contracts.NEO.Hash, neoOwner, util.Uint160{}, 1) diff --git a/pkg/core/blockchainer/blockchainer.go b/pkg/core/blockchainer/blockchainer.go index be17fd192..bdfdc06d8 100644 --- a/pkg/core/blockchainer/blockchainer.go +++ b/pkg/core/blockchainer/blockchainer.go @@ -23,7 +23,6 @@ type Blockchainer interface { GetConfig() config.ProtocolConfiguration AddHeaders(...*block.Header) error AddBlock(*block.Block) error - AddStateRoot(r *state.MPTRoot) error CalculateClaimable(h util.Uint160, endHeight uint32) (*big.Int, error) Close() IsTxStillRelevant(t *transaction.Transaction, txpool *mempool.Pool, isPartialTx bool) bool @@ -54,8 +53,7 @@ type Blockchainer interface { GetValidators() ([]*keys.PublicKey, error) GetStandByCommittee() keys.PublicKeys GetStandByValidators() keys.PublicKeys - GetStateProof(root util.Uint256, key []byte) ([][]byte, error) - GetStateRoot(height uint32) (*state.MPTRootState, error) + GetStateModule() StateRoot GetStorageItem(id int32, key []byte) state.StorageItem GetStorageItems(id int32) (map[string]state.StorageItem, error) GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) *vm.VM diff --git a/pkg/core/blockchainer/state_root.go b/pkg/core/blockchainer/state_root.go new file mode 100644 index 000000000..57c76c69f --- /dev/null +++ b/pkg/core/blockchainer/state_root.go @@ -0,0 +1,13 @@ +package blockchainer + +import ( + "github.com/nspcc-dev/neo-go/pkg/core/state" + "github.com/nspcc-dev/neo-go/pkg/util" +) + +// StateRoot represents local state root module. +type StateRoot interface { + CurrentLocalStateRoot() util.Uint256 + GetStateProof(root util.Uint256, key []byte) ([][]byte, error) + GetStateRoot(height uint32) (*state.MPTRoot, error) +} diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index af40a37f6..c8390aaf0 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/binary" "errors" - "fmt" iocore "io" "sort" @@ -42,12 +41,9 @@ type DAO interface { GetContractScriptHash(id int32) (util.Uint160, error) GetCurrentBlockHeight() (uint32, error) GetCurrentHeaderHeight() (i uint32, h util.Uint256, err error) - GetCurrentStateRootHeight() (uint32, error) GetHeaderHashes() ([]util.Uint256, error) GetNEP17Balances(acc util.Uint160) (*state.NEP17Balances, error) GetNEP17TransferLog(acc util.Uint160, index uint32) (*state.NEP17TransferLog, error) - GetStateRoot(height uint32) (*state.MPTRootState, error) - PutStateRoot(root *state.MPTRootState) error GetStorageItem(id int32, key []byte) state.StorageItem GetStorageItems(id int32) (map[string]state.StorageItem, error) GetStorageItemsWithPrefix(id int32, prefix []byte) (map[string]state.StorageItem, error) @@ -72,7 +68,6 @@ type DAO interface { // Simple is memCached wrapper around DB, simple DAO implementation. type Simple struct { - MPT *mpt.Trie Store *storage.MemCachedStore network netmode.Magic // stateRootInHeader specifies if block header contains state root. @@ -94,7 +89,6 @@ func (dao *Simple) GetBatch() *storage.MemBatch { // MemCachedStore around the current DAO Store. func (dao *Simple) GetWrapped() DAO { d := NewSimple(dao.Store, dao.network, dao.stateRootInHeader) - d.MPT = dao.MPT return d } @@ -289,75 +283,6 @@ func (dao *Simple) PutAppExecResult(aer *state.AppExecResult, buf *io.BufBinWrit // -- start storage item. -func makeStateRootKey(height uint32) []byte { - key := make([]byte, 5) - key[0] = byte(storage.DataMPT) - binary.LittleEndian.PutUint32(key[1:], height) - return key -} - -// InitMPT initializes MPT at the given height. -func (dao *Simple) InitMPT(height uint32, enableRefCount bool) error { - var gcKey = []byte{byte(storage.DataMPT), 1} - if height == 0 { - dao.MPT = mpt.NewTrie(nil, enableRefCount, dao.Store) - var val byte - if enableRefCount { - val = 1 - } - return dao.Store.Put(gcKey, []byte{val}) - } - var hasRefCount bool - if v, err := dao.Store.Get(gcKey); err == nil { - hasRefCount = v[0] != 0 - } - if hasRefCount != enableRefCount { - return fmt.Errorf("KeepOnlyLatestState setting mismatch: old=%v, new=%v", hasRefCount, enableRefCount) - } - r, err := dao.GetStateRoot(height) - if err != nil { - return err - } - dao.MPT = mpt.NewTrie(mpt.NewHashNode(r.Root), enableRefCount, dao.Store) - return nil -} - -// GetCurrentStateRootHeight returns current state root height. -func (dao *Simple) GetCurrentStateRootHeight() (uint32, error) { - key := []byte{byte(storage.DataMPT)} - val, err := dao.Store.Get(key) - if err != nil { - if err == storage.ErrKeyNotFound { - err = nil - } - return 0, err - } - return binary.LittleEndian.Uint32(val), nil -} - -// PutCurrentStateRootHeight updates current state root height. -func (dao *Simple) PutCurrentStateRootHeight(height uint32) error { - key := []byte{byte(storage.DataMPT)} - val := make([]byte, 4) - binary.LittleEndian.PutUint32(val, height) - return dao.Store.Put(key, val) -} - -// GetStateRoot returns state root of a given height. -func (dao *Simple) GetStateRoot(height uint32) (*state.MPTRootState, error) { - r := new(state.MPTRootState) - err := dao.GetAndDecode(r, makeStateRootKey(height)) - if err != nil { - return nil, err - } - return r, nil -} - -// PutStateRoot puts state root of a given height into the store. -func (dao *Simple) PutStateRoot(r *state.MPTRootState) error { - return dao.Put(r, makeStateRootKey(r.Index)) -} - // GetStorageItem returns StorageItem if it exists in the given store. func (dao *Simple) GetStorageItem(id int32, key []byte) state.StorageItem { b, err := dao.Store.Get(makeStorageItemKey(id, key)) @@ -672,12 +597,11 @@ func (dao *Simple) Persist() (int, error) { return dao.Store.Persist() } -// UpdateMPT updates MPT using storage items from the underlying memcached store. -func (dao *Simple) UpdateMPT() error { +// GetMPTBatch storage changes to be applied to MPT. +func (dao *Simple) GetMPTBatch() mpt.Batch { var b mpt.Batch dao.Store.MemoryStore.SeekAll([]byte{byte(storage.STStorage)}, func(k, v []byte) { b.Add(k[1:], v) }) - _, err := dao.MPT.PutBatch(b) - return err + return b } diff --git a/pkg/core/helper_test.go b/pkg/core/helper_test.go index 40b64f9b5..74b858b8d 100644 --- a/pkg/core/helper_test.go +++ b/pkg/core/helper_test.go @@ -87,7 +87,7 @@ func (bc *Blockchain) newBlock(txs ...*transaction.Transaction) *block.Block { } } if bc.config.StateRootInHeader { - sr, err := bc.GetStateRoot(bc.BlockHeight()) + sr, err := bc.GetStateModule().GetStateRoot(bc.BlockHeight()) if err != nil { panic(err) } diff --git a/pkg/core/native_management_test.go b/pkg/core/native_management_test.go index ceaf4b204..2a61a168f 100644 --- a/pkg/core/native_management_test.go +++ b/pkg/core/native_management_test.go @@ -10,6 +10,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/core/chaindump" "github.com/nspcc-dev/neo-go/pkg/core/state" + "github.com/nspcc-dev/neo-go/pkg/core/stateroot" "github.com/nspcc-dev/neo-go/pkg/core/storage" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/io" @@ -555,7 +556,8 @@ func TestContractDestroy(t *testing.T) { require.NoError(t, err) err = bc.dao.PutStorageItem(cs1.ID, []byte{1, 2, 3}, state.StorageItem{3, 2, 1}) require.NoError(t, err) - require.NoError(t, bc.dao.UpdateMPT()) + b := bc.dao.GetMPTBatch() + require.NoError(t, bc.GetStateModule().(*stateroot.Module).AddMPTBatch(bc.BlockHeight(), b)) t.Run("no contract", func(t *testing.T) { res, err := invokeContractMethod(bc, 1_00000000, mgmtHash, "destroy") diff --git a/pkg/core/prometheus.go b/pkg/core/prometheus.go index c849e3459..b81fb847d 100644 --- a/pkg/core/prometheus.go +++ b/pkg/core/prometheus.go @@ -30,14 +30,6 @@ var ( Namespace: "neogo", }, ) - //stateHeight prometheus metric. - stateHeight = prometheus.NewGauge( - prometheus.GaugeOpts{ - Help: "Current verified state height", - Name: "current_state_height", - Namespace: "neogo", - }, - ) ) func init() { @@ -59,7 +51,3 @@ func updateHeaderHeightMetric(hHeight int) { func updateBlockHeightMetric(bHeight uint32) { blockHeight.Set(float64(bHeight)) } - -func updateStateHeightMetric(sHeight uint32) { - stateHeight.Set(float64(sHeight)) -} diff --git a/pkg/core/state/mpt_root.go b/pkg/core/state/mpt_root.go index f2c890781..5fd716ff4 100644 --- a/pkg/core/state/mpt_root.go +++ b/pkg/core/state/mpt_root.go @@ -1,101 +1,54 @@ package state import ( - "encoding/json" - "errors" - "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/hash" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/util" ) -// MPTRootBase represents storage state root. -type MPTRootBase struct { - Version byte `json:"version"` - Index uint32 `json:"index"` - PrevHash util.Uint256 `json:"prehash"` - Root util.Uint256 `json:"stateroot"` -} - // MPTRoot represents storage state root together with sign info. type MPTRoot struct { - MPTRootBase + Version byte `json:"version"` + Index uint32 `json:"index"` + Root util.Uint256 `json:"stateroot"` Witness *transaction.Witness `json:"witness,omitempty"` } -// MPTRootStateFlag represents verification state of the state root. -type MPTRootStateFlag byte - -// Possible verification states of MPTRoot. -const ( - Unverified MPTRootStateFlag = 0x00 - Verified MPTRootStateFlag = 0x01 - Invalid MPTRootStateFlag = 0x03 -) - -// MPTRootState represents state root together with its verification state. -type MPTRootState struct { - MPTRoot `json:"stateroot"` - Flag MPTRootStateFlag `json:"flag"` -} - -// EncodeBinary implements io.Serializable. -func (s *MPTRootState) EncodeBinary(w *io.BinWriter) { - w.WriteB(byte(s.Flag)) - s.MPTRoot.EncodeBinary(w) -} - -// DecodeBinary implements io.Serializable. -func (s *MPTRootState) DecodeBinary(r *io.BinReader) { - s.Flag = MPTRootStateFlag(r.ReadB()) - s.MPTRoot.DecodeBinary(r) -} - // GetSignedPart returns part of MPTRootBase which needs to be signed. -func (s *MPTRootBase) GetSignedPart() []byte { +func (s *MPTRoot) GetSignedPart() []byte { buf := io.NewBufBinWriter() - s.EncodeBinary(buf.BinWriter) + s.EncodeBinaryUnsigned(buf.BinWriter) return buf.Bytes() } // GetSignedHash returns hash of MPTRootBase which needs to be signed. -func (s *MPTRootBase) GetSignedHash() util.Uint256 { - buf := io.NewBufBinWriter() - s.EncodeBinary(buf.BinWriter) - return hash.Sha256(buf.Bytes()) -} - -// Equals checks if s == other. -func (s *MPTRootBase) Equals(other *MPTRootBase) bool { - return s.Version == other.Version && s.Index == other.Index && - s.PrevHash.Equals(other.PrevHash) && s.Root.Equals(other.Root) +func (s *MPTRoot) GetSignedHash() util.Uint256 { + return hash.Sha256(s.GetSignedPart()) } // Hash returns hash of s. -func (s *MPTRootBase) Hash() util.Uint256 { +func (s *MPTRoot) Hash() util.Uint256 { return hash.DoubleSha256(s.GetSignedPart()) } -// DecodeBinary implements io.Serializable. -func (s *MPTRootBase) DecodeBinary(r *io.BinReader) { +// DecodeBinaryUnsigned decodes hashable part of state root. +func (s *MPTRoot) DecodeBinaryUnsigned(r *io.BinReader) { s.Version = r.ReadB() s.Index = r.ReadU32LE() - s.PrevHash.DecodeBinary(r) s.Root.DecodeBinary(r) } -// EncodeBinary implements io.Serializable. -func (s *MPTRootBase) EncodeBinary(w *io.BinWriter) { +// EncodeBinaryUnsigned encodes hashable part of state root.. +func (s *MPTRoot) EncodeBinaryUnsigned(w *io.BinWriter) { w.WriteB(s.Version) w.WriteU32LE(s.Index) - s.PrevHash.EncodeBinary(w) s.Root.EncodeBinary(w) } // DecodeBinary implements io.Serializable. func (s *MPTRoot) DecodeBinary(r *io.BinReader) { - s.MPTRootBase.DecodeBinary(r) + s.DecodeBinaryUnsigned(r) var ws []transaction.Witness r.ReadArray(&ws, 1) @@ -106,48 +59,10 @@ func (s *MPTRoot) DecodeBinary(r *io.BinReader) { // EncodeBinary implements io.Serializable. func (s *MPTRoot) EncodeBinary(w *io.BinWriter) { - s.MPTRootBase.EncodeBinary(w) + s.EncodeBinaryUnsigned(w) if s.Witness == nil { w.WriteVarUint(0) } else { w.WriteArray([]*transaction.Witness{s.Witness}) } } - -// String implements fmt.Stringer. -func (f MPTRootStateFlag) String() string { - switch f { - case Unverified: - return "Unverified" - case Verified: - return "Verified" - case Invalid: - return "Invalid" - default: - return "" - } -} - -// MarshalJSON implements json.Marshaler. -func (f MPTRootStateFlag) MarshalJSON() ([]byte, error) { - return []byte(`"` + f.String() + `"`), nil -} - -// UnmarshalJSON implements json.Unmarshaler. -func (f *MPTRootStateFlag) UnmarshalJSON(data []byte) error { - var s string - if err := json.Unmarshal(data, &s); err != nil { - return err - } - switch s { - case "Unverified": - *f = Unverified - case "Verified": - *f = Verified - case "Invalid": - *f = Invalid - default: - return errors.New("unknown flag") - } - return nil -} diff --git a/pkg/core/state/mpt_root_test.go b/pkg/core/state/mpt_root_test.go index da9ff9c46..eaf5c45dd 100644 --- a/pkg/core/state/mpt_root_test.go +++ b/pkg/core/state/mpt_root_test.go @@ -14,12 +14,9 @@ import ( func testStateRoot() *MPTRoot { return &MPTRoot{ - MPTRootBase: MPTRootBase{ - Version: byte(rand.Uint32()), - Index: rand.Uint32(), - PrevHash: random.Uint256(), - Root: random.Uint256(), - }, + Version: byte(rand.Uint32()), + Index: rand.Uint32(), + Root: random.Uint256(), } } @@ -36,64 +33,27 @@ func TestStateRoot_Serializable(t *testing.T) { }) } -func TestStateRootEquals(t *testing.T) { - r1 := testStateRoot() - r2 := *r1 - require.True(t, r1.Equals(&r2.MPTRootBase)) - - r2.MPTRootBase.Index++ - require.False(t, r1.Equals(&r2.MPTRootBase)) -} - -func TestMPTRootState_Serializable(t *testing.T) { - rs := &MPTRootState{ - MPTRoot: *testStateRoot(), - Flag: 0x04, - } - rs.MPTRoot.Witness = &transaction.Witness{ - InvocationScript: random.Bytes(10), - VerificationScript: random.Bytes(11), - } - testserdes.EncodeDecodeBinary(t, rs, new(MPTRootState)) -} - -func TestMPTRootStateUnverifiedByDefault(t *testing.T) { - var r MPTRootState - require.Equal(t, Unverified, r.Flag) -} - func TestMPTRoot_MarshalJSON(t *testing.T) { t.Run("Good", func(t *testing.T) { r := testStateRoot() - rs := &MPTRootState{ - MPTRoot: *r, - Flag: Verified, - } - testserdes.MarshalUnmarshalJSON(t, rs, new(MPTRootState)) + testserdes.MarshalUnmarshalJSON(t, r, new(MPTRoot)) }) t.Run("Compatibility", func(t *testing.T) { js := []byte(`{ - "flag": "Unverified", - "stateroot": { "version": 1, "index": 3000000, - "prehash": "0x4f30f43af8dd2262fc331c45bfcd9066ebbacda204e6e81371cbd884fe7d6c90", "stateroot": "0xb2fd7e368a848ef70d27cf44940a35237333ed05f1d971c9408f0eb285e0b6f3" - }}`) + }`) - rs := new(MPTRootState) + rs := new(MPTRoot) require.NoError(t, json.Unmarshal(js, &rs)) require.EqualValues(t, 1, rs.Version) require.EqualValues(t, 3000000, rs.Index) require.Nil(t, rs.Witness) - u, err := util.Uint256DecodeStringLE("4f30f43af8dd2262fc331c45bfcd9066ebbacda204e6e81371cbd884fe7d6c90") - require.NoError(t, err) - require.Equal(t, u, rs.PrevHash) - - u, err = util.Uint256DecodeStringLE("b2fd7e368a848ef70d27cf44940a35237333ed05f1d971c9408f0eb285e0b6f3") + u, err := util.Uint256DecodeStringLE("b2fd7e368a848ef70d27cf44940a35237333ed05f1d971c9408f0eb285e0b6f3") require.NoError(t, err) require.Equal(t, u, rs.Root) }) diff --git a/pkg/core/stateroot/module.go b/pkg/core/stateroot/module.go new file mode 100644 index 000000000..9bf646fca --- /dev/null +++ b/pkg/core/stateroot/module.go @@ -0,0 +1,98 @@ +package stateroot + +import ( + "fmt" + + "github.com/nspcc-dev/neo-go/pkg/core/blockchainer" + "github.com/nspcc-dev/neo-go/pkg/core/mpt" + "github.com/nspcc-dev/neo-go/pkg/core/state" + "github.com/nspcc-dev/neo-go/pkg/core/storage" + "github.com/nspcc-dev/neo-go/pkg/util" + "go.uber.org/atomic" + "go.uber.org/zap" +) + +type ( + // Module represents module for local processing of state roots. + Module struct { + Store *storage.MemCachedStore + mpt *mpt.Trie + bc blockchainer.Blockchainer + log *zap.Logger + + localHeight atomic.Uint32 + validatedHeight atomic.Uint32 + currentLocal atomic.Value + } +) + +// NewModule returns new instance of stateroot module. +func NewModule(bc blockchainer.Blockchainer, log *zap.Logger, s *storage.MemCachedStore) *Module { + return &Module{ + bc: bc, + log: log, + Store: s, + } +} + +// GetStateProof returns proof of having key in the MPT with the specified root. +func (s *Module) GetStateProof(root util.Uint256, key []byte) ([][]byte, error) { + tr := mpt.NewTrie(mpt.NewHashNode(root), false, storage.NewMemCachedStore(s.Store)) + return tr.GetProof(key) +} + +// GetStateRoot returns state root for a given height. +func (s *Module) GetStateRoot(height uint32) (*state.MPTRoot, error) { + return s.getStateRoot(makeStateRootKey(height)) +} + +// CurrentLocalStateRoot returns hash of the local state root. +func (s *Module) CurrentLocalStateRoot() util.Uint256 { + return s.currentLocal.Load().(util.Uint256) +} + +// Init initializes state root module at the given height. +func (s *Module) Init(height uint32, enableRefCount bool) error { + var gcKey = []byte{byte(storage.DataMPT), prefixGC} + if height == 0 { + s.mpt = mpt.NewTrie(nil, enableRefCount, s.Store) + var val byte + if enableRefCount { + val = 1 + } + s.currentLocal.Store(util.Uint256{}) + return s.Store.Put(gcKey, []byte{val}) + } + var hasRefCount bool + if v, err := s.Store.Get(gcKey); err == nil { + hasRefCount = v[0] != 0 + } + if hasRefCount != enableRefCount { + return fmt.Errorf("KeepOnlyLatestState setting mismatch: old=%v, new=%v", hasRefCount, enableRefCount) + } + r, err := s.getStateRoot(makeStateRootKey(height)) + if err != nil { + return err + } + s.currentLocal.Store(r.Root) + s.localHeight.Store(r.Index) + s.mpt = mpt.NewTrie(mpt.NewHashNode(r.Root), enableRefCount, s.Store) + return nil +} + +// AddMPTBatch updates using provided batch. +func (s *Module) AddMPTBatch(index uint32, b mpt.Batch) error { + if _, err := s.mpt.PutBatch(b); err != nil { + return err + } + s.mpt.Flush() + err := s.addLocalStateRoot(&state.MPTRoot{ + Index: index, + Root: s.mpt.StateRoot(), + }) + if err != nil { + return err + } + _, err = s.Store.Persist() + return err +} diff --git a/pkg/core/stateroot/prometheus.go b/pkg/core/stateroot/prometheus.go new file mode 100644 index 000000000..cd6346404 --- /dev/null +++ b/pkg/core/stateroot/prometheus.go @@ -0,0 +1,20 @@ +package stateroot + +import "github.com/prometheus/client_golang/prometheus" + +// stateHeight prometheus metric. +var stateHeight = prometheus.NewGauge( + prometheus.GaugeOpts{ + Help: "Current verified state height", + Name: "current_state_height", + Namespace: "neogo", + }, +) + +func init() { + prometheus.MustRegister(stateHeight) +} + +func updateStateHeightMetric(sHeight uint32) { + stateHeight.Set(float64(sHeight)) +} diff --git a/pkg/core/stateroot/store.go b/pkg/core/stateroot/store.go new file mode 100644 index 000000000..cc43bd0cf --- /dev/null +++ b/pkg/core/stateroot/store.go @@ -0,0 +1,56 @@ +package stateroot + +import ( + "encoding/binary" + + "github.com/nspcc-dev/neo-go/pkg/core/state" + "github.com/nspcc-dev/neo-go/pkg/core/storage" + "github.com/nspcc-dev/neo-go/pkg/io" +) + +const ( + prefixGC = 0x01 + prefixLocal = 0x02 +) + +func (s *Module) addLocalStateRoot(sr *state.MPTRoot) error { + key := makeStateRootKey(sr.Index) + if err := s.putStateRoot(key, sr); err != nil { + return err + } + + data := make([]byte, 4) + binary.LittleEndian.PutUint32(data, sr.Index) + if err := s.Store.Put([]byte{byte(storage.DataMPT), prefixLocal}, data); err != nil { + return err + } + s.currentLocal.Store(sr.Root) + s.localHeight.Store(sr.Index) + updateStateHeightMetric(sr.Index) + return nil +} + +func (s *Module) putStateRoot(key []byte, sr *state.MPTRoot) error { + w := io.NewBufBinWriter() + sr.EncodeBinary(w.BinWriter) + return s.Store.Put(key, w.Bytes()) +} + +func (s *Module) getStateRoot(key []byte) (*state.MPTRoot, error) { + data, err := s.Store.Get(key) + if err != nil { + return nil, err + } + + sr := new(state.MPTRoot) + r := io.NewBinReaderFromBuf(data) + sr.DecodeBinary(r) + return sr, r.Err +} + +func makeStateRootKey(index uint32) []byte { + key := make([]byte, 5) + key[0] = byte(storage.DataMPT) + binary.BigEndian.PutUint32(key, index) + return key +} diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index db22543d7..fbc1d4a99 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -833,7 +833,7 @@ func (s *Server) getProof(ps request.Params) (interface{}, *response.Error) { return nil, response.ErrInvalidParams } skey := makeStorageKey(cs.ID, key) - proof, err := s.chain.GetStateProof(root, skey) + proof, err := s.chain.GetStateModule().GetStateProof(root, skey) return &result.GetProof{ Result: result.ProofWithKey{ Key: skey, @@ -884,16 +884,16 @@ func (s *Server) getStateRoot(ps request.Params) (interface{}, *response.Error) if p == nil { return nil, response.NewRPCError("Invalid parameter.", "", nil) } - var rt *state.MPTRootState + var rt *state.MPTRoot var h util.Uint256 height, err := p.GetInt() if err == nil { - rt, err = s.chain.GetStateRoot(uint32(height)) + rt, err = s.chain.GetStateModule().GetStateRoot(uint32(height)) } else if h, err = p.GetUint256(); err == nil { var hdr *block.Header hdr, err = s.chain.GetHeader(h) if err == nil { - rt, err = s.chain.GetStateRoot(hdr.Index) + rt, err = s.chain.GetStateModule().GetStateRoot(hdr.Index) } } if err != nil { diff --git a/pkg/rpc/server/server_test.go b/pkg/rpc/server/server_test.go index c643aa790..c7f430f3c 100644 --- a/pkg/rpc/server/server_test.go +++ b/pkg/rpc/server/server_test.go @@ -1287,7 +1287,7 @@ func testRPCProtocol(t *testing.T, doRPCCall func(string, string, *testing.T) [] }) }) t.Run("getproof", func(t *testing.T) { - r, err := chain.GetStateRoot(3) + r, err := chain.GetStateModule().GetStateRoot(3) require.NoError(t, err) rpc := fmt.Sprintf(`{"jsonrpc": "2.0", "id": 1, "method": "getproof", "params": ["%s", "%s", "%x"]}`, @@ -1316,11 +1316,11 @@ func testRPCProtocol(t *testing.T, doRPCCall func(string, string, *testing.T) [] body := doRPCCall(rpc, httpSrv.URL, t) rawRes := checkErrGetResult(t, body, false) - res := new(state.MPTRootState) + res := new(state.MPTRoot) require.NoError(t, json.Unmarshal(rawRes, res)) require.NotEqual(t, util.Uint256{}, res.Root) // be sure this test uses valid height - expected, err := e.chain.GetStateRoot(5) + expected, err := e.chain.GetStateModule().GetStateRoot(5) require.NoError(t, err) require.Equal(t, expected, res) } diff --git a/scripts/gendump/main.go b/scripts/gendump/main.go index 8b93aea5a..977cae05a 100644 --- a/scripts/gendump/main.go +++ b/scripts/gendump/main.go @@ -175,7 +175,7 @@ func newBlock(bc *core.Blockchain, lastBlock *block.Block, script []byte, txs .. Transactions: txs, } if bc.GetConfig().StateRootInHeader { - sr, err := bc.GetStateRoot(bc.BlockHeight()) + sr, err := bc.GetStateModule().GetStateRoot(bc.BlockHeight()) if err != nil { return nil, err }