core: add some mechanism to drop bad blocks from cache

If we can't persist anything from our cache there is something wrong with
it. Try to remediate some possible problems.

pkg/core/blockchain.go

@@ -429,6 +429,15 @@ func (bc *Blockchain) persist(ctx context.Context) (err error) {
 			"blockHeight":  bc.BlockHeight(),
 			"took":         time.Since(start),
 		}).Info("blockchain persist completed")
+	} else {
+		// So we have some blocks in cache but can't persist them?
+		// Either there are some stale blocks there or the other way
+		// around (which was seen in practice) --- there are some fresh
+		// blocks that we can't persist yet. Some of the latter can be useful
+		// or can be bogus (higher than the header height we expect at
+		// the moment). So try to reap oldies and strange newbies, if
+		// there are any.
+		bc.blockCache.ReapStrangeBlocks(bc.BlockHeight(), bc.HeaderHeight())
 	}
 
 	return

pkg/core/cache.go

@@ -71,3 +71,17 @@ func (c *Cache) Delete(h util.Uint256) {
 	defer c.lock.Unlock()
 	delete(c.m, h)
 }
+
+// ReapStrangeBlocks drops blocks from cache that don't fit into the
+// blkHeight-headHeight interval. Cache should only contain blocks that we
+// expect to get and store.
+func (c *Cache) ReapStrangeBlocks(blkHeight, headHeight uint32) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	for i, b := range c.m {
+		block, ok := b.(*Block)
+		if ok && (block.Index < blkHeight || block.Index > headHeight) {
+			delete(c.m, i)
+		}
+	}
+}