core: implement basic GC for value-based storage scheme

The key idea here is that even though we can't ensure MPT code won't make the
node active again we can order the changes made to the persistent store in
such a way that it practically doesn't matter. What happens is:
 * after persist if it's time to collect our garbage we do it synchronously
   right in the same thread working the underlying persistent store directly
 * all the other node code doesn't see much of it, it works with bc.dao or
   layers above it
 * if MPT doesn't find some stale deactivated node in the storage it's OK,
   it'll recreate it in bc.dao
 * if MPT finds it and activates it, it's OK too, bc.dao will store it
 * while GC is being performed nothing else changes the persistent store
 * all subsequent bc.dao persists only happen after the GC is completed which
   means that any changes to the (potentially) deleted nodes have a priority,
   it's OK for GC to delete something that'll be recreated with the next
   persist cycle

Otherwise it's a simple scheme with node status/last active height stored in
the value. Preliminary tests show that it works ~18% worse than the simple
KeepOnlyLatest scheme, but this seems to be the best result so far.

Fixes #2095.

pkg/core/blockchain.go

@@ -47,6 +47,7 @@ const (
 	version          = "0.2.2"
 
 	defaultInitialGAS                      = 52000000_00000000
+	defaultGCPeriod                        = 10000
 	defaultMemPoolSize                     = 50000
 	defaultP2PNotaryRequestPayloadPoolSize = 1000
 	defaultMaxBlockSize                    = 262144
@@ -124,6 +125,9 @@ type Blockchain struct {
 	// are directly from underlying persistent store.
 	persistent *dao.Simple
 
+	// Underlying persistent store.
+	store storage.Store
+
 	// Current index/height of the highest block.
 	// Read access should always be called by BlockHeight().
 	// Write access should only happen in storeBlock().
@@ -245,6 +249,10 @@ func NewBlockchain(s storage.Store, cfg config.ProtocolConfiguration, log *zap.L
 				zap.Int("StateSyncInterval", cfg.StateSyncInterval))
 		}
 	}
+	if cfg.RemoveUntraceableBlocks && cfg.GarbageCollectionPeriod == 0 {
+		cfg.GarbageCollectionPeriod = defaultGCPeriod
+		log.Info("GarbageCollectionPeriod is not set or wrong, using default value", zap.Uint32("GarbageCollectionPeriod", cfg.GarbageCollectionPeriod))
+	}
 	if len(cfg.NativeUpdateHistories) == 0 {
 		cfg.NativeUpdateHistories = map[string][]uint32{}
 		log.Info("NativeActivations are not set, using default values")
@@ -253,6 +261,7 @@ func NewBlockchain(s storage.Store, cfg config.ProtocolConfiguration, log *zap.L
 		config:      cfg,
 		dao:         dao.NewSimple(s, cfg.StateRootInHeader, cfg.P2PSigExtensions),
 		persistent:  dao.NewSimple(s, cfg.StateRootInHeader, cfg.P2PSigExtensions),
+		store:       s,
 		stopCh:      make(chan struct{}),
 		runToExitCh: make(chan struct{}),
 		memPool:     mempool.New(cfg.MemPoolSize, 0, false),
@@ -647,12 +656,21 @@ func (bc *Blockchain) Run() {
 		case <-bc.stopCh:
 			return
 		case <-persistTimer.C:
+			var oldPersisted uint32
+			var gcDur time.Duration
+
+			if bc.config.RemoveUntraceableBlocks {
+				oldPersisted = atomic.LoadUint32(&bc.persistedHeight)
+			}
 			dur, err := bc.persist(nextSync)
 			if err != nil {
 				bc.log.Warn("failed to persist blockchain", zap.Error(err))
 			}
+			if bc.config.RemoveUntraceableBlocks {
+				gcDur = bc.tryRunGC(oldPersisted)
+			}
 			nextSync = dur > persistInterval*2
-			interval := persistInterval - dur
+			interval := persistInterval - dur - gcDur
 			if interval <= 0 {
 				interval = time.Microsecond // Reset doesn't work with zero value
 			}
@@ -661,6 +679,35 @@ func (bc *Blockchain) Run() {
 	}
 }
 
+func (bc *Blockchain) tryRunGC(old uint32) time.Duration {
+	var dur time.Duration
+
+	new := atomic.LoadUint32(&bc.persistedHeight)
+	var tgtBlock = int64(new)
+
+	tgtBlock -= int64(bc.config.MaxTraceableBlocks)
+	if bc.config.P2PStateExchangeExtensions {
+		syncP := new / uint32(bc.config.StateSyncInterval)
+		syncP--
+		syncP *= uint32(bc.config.StateSyncInterval)
+		if tgtBlock > int64(syncP) {
+			tgtBlock = int64(syncP)
+		}
+	}
+	// Always round to the GCP.
+	tgtBlock /= int64(bc.config.GarbageCollectionPeriod)
+	tgtBlock *= int64(bc.config.GarbageCollectionPeriod)
+	// Count periods.
+	old /= bc.config.GarbageCollectionPeriod
+	new /= bc.config.GarbageCollectionPeriod
+	if tgtBlock > int64(bc.config.GarbageCollectionPeriod) && new != old {
+		tgtBlock /= int64(bc.config.GarbageCollectionPeriod)
+		tgtBlock *= int64(bc.config.GarbageCollectionPeriod)
+		dur = bc.stateRoot.GC(uint32(tgtBlock), bc.store)
+	}
+	return dur
+}
+
 // notificationDispatcher manages subscription to events and broadcasts new events.
 func (bc *Blockchain) notificationDispatcher() {
 	var (