From 51f405471ebb6566bbf33efce8d8e0f0a116a050 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 17 Aug 2021 18:16:10 +0300 Subject: [PATCH] core: remove outdated blocks/txs/AERs/MPT nodes during state sync Before state sync process can be started, outdated MPT nodes should be removed from storage. After state sync is completed, outdated blocks/transactions/AERs should also be removed. --- pkg/core/blockchain.go | 15 ++++++++++- pkg/core/blockchainer/state_root.go | 1 + pkg/core/stateroot/module.go | 42 +++++++++++++++++++++++++++++ pkg/core/statesync/module.go | 31 ++++++++++++++++++--- 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 1b3a99370..3aa37943d 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -431,7 +431,8 @@ func (bc *Blockchain) JumpToState(module blockchainer.StateSync) error { if err != nil { return fmt.Errorf("failed to get current block: %w", err) } - err = bc.dao.StoreAsCurrentBlock(block, nil) + writeBuf := io.NewBufBinWriter() + err = bc.dao.StoreAsCurrentBlock(block, writeBuf) if err != nil { return fmt.Errorf("failed to store current block: %w", err) } @@ -464,6 +465,18 @@ func (bc *Blockchain) JumpToState(module blockchainer.StateSync) error { return fmt.Errorf("failed to update extensible whitelist: %w", err) } + // After current state is updated, we need to remove outdated state-related data if so. + // The only outdated data we might have is genesis-related data, so check it. + if p-bc.config.MaxTraceableBlocks > 0 { + cache := bc.dao.GetWrapped() + writeBuf.Reset() + err = cache.DeleteBlock(bc.headerHashes[0], writeBuf) + if err != nil { + return fmt.Errorf("failed to remove outdated state data for the genesis block: %w", err) + } + // TODO: remove NEP17 transfers and NEP17 transfer info for genesis block, #2096 related. + } + updateBlockHeightMetric(p) return nil } diff --git a/pkg/core/blockchainer/state_root.go b/pkg/core/blockchainer/state_root.go index 64f7d8b7c..9a540bda8 100644 --- a/pkg/core/blockchainer/state_root.go +++ b/pkg/core/blockchainer/state_root.go @@ -9,6 +9,7 @@ import ( // StateRoot represents local state root module. type StateRoot interface { AddStateRoot(root *state.MPTRoot) error + CleanStorage() error CurrentLocalHeight() uint32 CurrentLocalStateRoot() util.Uint256 CurrentValidatedHeight() uint32 diff --git a/pkg/core/stateroot/module.go b/pkg/core/stateroot/module.go index 20a3396e9..838e4717a 100644 --- a/pkg/core/stateroot/module.go +++ b/pkg/core/stateroot/module.go @@ -13,6 +13,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/storage" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.uber.org/atomic" "go.uber.org/zap" ) @@ -114,6 +115,47 @@ func (s *Module) Init(height uint32, enableRefCount bool) error { return nil } +// CleanStorage removes all MPT-related data from the storage (MPT nodes, validated stateroots) +// except local stateroot for the current height and GC flag. This method is aimed to clean +// outdated MPT data before state sync process can be started. +// Note: this method is aimed to be called for genesis block only, an error is returned otherwice. +func (s *Module) CleanStorage() error { + if s.localHeight.Load() != 0 { + return fmt.Errorf("can't clean MPT data for non-genesis block: expected local stateroot height 0, got %d", s.localHeight.Load()) + } + gcKey := []byte{byte(storage.DataMPT), prefixGC} + gcVal, err := s.Store.Get(gcKey) + if err != nil { + return fmt.Errorf("failed to get GC flag: %w", err) + } + // + b := s.Store.Batch() + s.Store.Seek([]byte{byte(storage.DataMPT)}, func(k, _ []byte) { + // Must copy here, #1468. + key := slice.Copy(k) + b.Delete(key) + }) + err = s.Store.PutBatch(b) + if err != nil { + return fmt.Errorf("failed to remove outdated MPT-reated items: %w", err) + } + err = s.Store.Put(gcKey, gcVal) + if err != nil { + return fmt.Errorf("failed to store GC flag: %w", err) + } + currentLocal := s.currentLocal.Load().(util.Uint256) + if !currentLocal.Equals(util.Uint256{}) { + err := s.addLocalStateRoot(s.Store, &state.MPTRoot{ + Index: s.localHeight.Load(), + Root: currentLocal, + }) + if err != nil { + return fmt.Errorf("failed to store current local stateroot: %w", err) + } + } + return nil +} + // JumpToState performs jump to the state specified by given stateroot index. func (s *Module) JumpToState(sr *state.MPTRoot, enableRefCount bool) error { if err := s.addLocalStateRoot(s.Store, sr); err != nil { diff --git a/pkg/core/statesync/module.go b/pkg/core/statesync/module.go index 507fbc373..80939de1c 100644 --- a/pkg/core/statesync/module.go +++ b/pkg/core/statesync/module.go @@ -120,10 +120,33 @@ func (s *Module) Init(currChainHeight uint32) error { if err == nil && pOld >= p-s.syncInterval { // old point is still valid, so try to resync states for this point. p = pOld - } else if s.bc.BlockHeight() > p-2*s.syncInterval { - // chain has already been synchronised up to old state sync point and regular blocks processing was started - s.syncStage = inactive - return nil + } else { + if s.bc.BlockHeight() > p-2*s.syncInterval { + // chain has already been synchronised up to old state sync point and regular blocks processing was started. + // Current block height is enough to start regular blocks processing. + s.syncStage = inactive + return nil + } + if err == nil { + // pOld was found, it is outdated, and chain wasn't completely synchronised for pOld. Need to drop the db. + return fmt.Errorf("state sync point %d is found in the storage, "+ + "but sync process wasn't completed and point is outdated. Please, drop the database manually and restart the node to run state sync process", pOld) + } + if s.bc.BlockHeight() != 0 { + // pOld wasn't found, but blocks processing was started in a regular manner and latest stored block is too outdated + // to start regular blocks processing again. Need to drop the db. + return fmt.Errorf("current chain's height is too low to start regular blocks processing from the oldest sync point %d. "+ + "Please, drop the database manually and restart the node to run state sync process", p-s.syncInterval) + } + + // We've reached this point, so chain has genesis block only. As far as we can't ruin + // current chain's state until new state is completely fetched, outdated state-related data + // will be removed from storage during (*Blockchain).JumpToState(...) execution. + // All we need to do right now is to remove genesis-related MPT nodes. + err = s.bc.GetStateModule().CleanStorage() + if err != nil { + return fmt.Errorf("failed to remove outdated MPT data from storage: %w", err) + } } s.syncPoint = p