storage: allow accessing MemCachedStore during Persist

Persist by its definition doesn't change MemCachedStore visible state, all KV
pairs that were acessible via it before Persist remain accessible after
Persist. The only thing it does is flushing of the current set of KV pairs
from memory to peristent store. To do that it needs read-only access to the
current KV pair set, but technically it then replaces maps, so we have to use
full write lock which makes MemCachedStore inaccessible for the duration of
Persist. And Persist can take a lot of time, it's about disk access for
regular DBs.

What we do here is we create new in-memory maps for MemCachedStore before
flushing old ones to the persistent store. Then a fake persistent store is
created which actually is a MemCachedStore with old maps, so it has exactly
the same visible state. This Store is never accessed for writes, so we can
read it without taking any internal locks and at the same time we no longer
need write locks for original MemCachedStore, we're not using it. All of this
makes it possible to use MemCachedStore as normally reads are handled going
down to whatever level is needed and writes are handled by new maps. So while
Persist for (*Blockchain).dao does its most time-consuming work we can process
other blocks (reading data for transactions and persisting storeBlock caches
to (*Blockchain).dao).

The change was tested for performance with neo-bench (single node, 10 workers,
LevelDB) on two machines and block dump processing (RC4 testnet up to 62800
with VerifyBlocks set to false) on i7-8565U.

Reference results (bbe4e9cd7b):

Ryzen 9 5950X:
RPS     23616.969 22817.086 23222.378  ≈ 23218   ± 1.72%
TPS     23047.316 22608.578 22735.540  ≈ 22797   ± 0.99%
CPU %      23.434    25.553    23.848  ≈    24.3 ± 4.63%
Mem MB    600.636   503.060   582.043  ≈   562   ± 9.22%

Core i7-8565U:
RPS     6594.007 6499.501 6572.902  ≈ 6555   ± 0.76%
TPS     6561.680 6444.545 6510.120  ≈ 6505   ± 0.90%
CPU %     58.452   60.568   62.474    ≈ 60.5 ± 3.33%
Mem MB   234.893  285.067  269.081   ≈ 263   ± 9.75%

DB restore:
real    0m22.237s 0m23.471s 0m23.409s  ≈ 23.04 ± 3.02%
user    0m35.435s 0m38.943s 0m39.247s  ≈ 37.88 ± 5.59%
sys      0m3.085s  0m3.360s  0m3.144s  ≈  3.20 ± 4.53%

After the change:

Ryzen 9 5950X:
RPS     27747.349 27407.726 27520.210  ≈ 27558   ± 0.63%  ↑ 18.69%
TPS     26992.010 26993.468 27010.966  ≈ 26999   ± 0.04%  ↑ 18.43%
CPU %      28.928    28.096    29.105  ≈    28.7 ± 1.88%  ↑ 18.1%
Mem MB    760.385   726.320   756.118  ≈   748   ± 2.48%  ↑ 33.10%

Core i7-8565U:
RPS     7783.229 7628.409 7542.340  ≈ 7651   ± 1.60%  ↑ 16.72%
TPS     7708.436 7607.397 7489.459  ≈ 7602   ± 1.44%  ↑ 16.85%
CPU %     74.899   71.020   72.697  ≈   72.9 ± 2.67%  ↑ 20.50%
Mem MB   438.047  436.967  416.350  ≈  430   ± 2.84%  ↑ 63.50%

DB restore:
real    0m20.838s 0m21.895s 0m21.794s  ≈ 21.51 ± 2.71%  ↓ 6.64%
user    0m39.091s 0m40.565s 0m41.493s  ≈ 40.38 ± 3.00%  ↑ 6.60%
sys      0m3.184s  0m2.923s  0m3.062s  ≈  3.06 ± 4.27%  ↓ 4.38%

It obviously uses more memory now and utilizes CPU more aggressively, but at
the same time it allows to improve all relevant metrics and finally reach a
situation where we process 50K transactions in less than second on Ryzen 9
5950X (going higher than 25K TPS). The other observation is much more stable
block time, on Ryzen 9 it's as close to 1 second as it could be.

pkg/core/storage/memcached_store.go

@@ -1,10 +1,14 @@
 package storage
 
+import "sync"
+
 // MemCachedStore is a wrapper around persistent store that caches all changes
 // being made for them to be later flushed in one batch.
 type MemCachedStore struct {
 	MemoryStore
 
+	// plock protects Persist from double entrance.
+	plock sync.Mutex
 	// Persistent Store.
 	ps Store
 }
@@ -96,45 +100,73 @@ func (s *MemCachedStore) Persist() (int, error) {
 	var err error
 	var keys, dkeys int
 
+	s.plock.Lock()
+	defer s.plock.Unlock()
 	s.mut.Lock()
-	defer s.mut.Unlock()
 
 	keys = len(s.mem)
 	dkeys = len(s.del)
 	if keys == 0 && dkeys == 0 {
+		s.mut.Unlock()
 		return 0, nil
 	}
 
-	memStore, ok := s.ps.(*MemoryStore)
+	// tempstore technically copies current s in lower layer while real s
+	// starts using fresh new maps. This tempstore is only known here and
+	// nothing ever changes it, therefore accesses to it (reads) can go
+	// unprotected while writes are handled by s proper.
+	var tempstore = &MemCachedStore{MemoryStore: MemoryStore{mem: s.mem, del: s.del}, ps: s.ps}
+	s.ps = tempstore
+	s.mem = make(map[string][]byte)
+	s.del = make(map[string]bool)
+	s.mut.Unlock()
+
+	memStore, ok := tempstore.ps.(*MemoryStore)
 	if !ok {
-		memCachedStore, ok := s.ps.(*MemCachedStore)
+		memCachedStore, ok := tempstore.ps.(*MemCachedStore)
 		if ok {
 			memStore = &memCachedStore.MemoryStore
 		}
 	}
 	if memStore != nil {
 		memStore.mut.Lock()
-		for k := range s.mem {
+		for k := range tempstore.mem {
-			memStore.put(k, s.mem[k])
+			memStore.put(k, tempstore.mem[k])
 		}
-		for k := range s.del {
+		for k := range tempstore.del {
 			memStore.drop(k)
 		}
 		memStore.mut.Unlock()
 	} else {
-		batch := s.ps.Batch()
+		batch := tempstore.ps.Batch()
-		for k := range s.mem {
+		for k := range tempstore.mem {
-			batch.Put([]byte(k), s.mem[k])
+			batch.Put([]byte(k), tempstore.mem[k])
 		}
-		for k := range s.del {
+		for k := range tempstore.del {
 			batch.Delete([]byte(k))
 		}
-		err = s.ps.PutBatch(batch)
+		err = tempstore.ps.PutBatch(batch)
 	}
+	s.mut.Lock()
 	if err == nil {
-		s.mem = make(map[string][]byte)
+		// tempstore.mem and tempstore.del are completely flushed now
-		s.del = make(map[string]bool)
+		// to tempstore.ps, so all KV pairs are the same and this
+		// substitution has no visible effects.
+		s.ps = tempstore.ps
+	} else {
+		// We're toast. We'll try to still keep proper state, but OOM
+		// killer will get to us eventually.
+		for k := range s.mem {
+			tempstore.put(k, s.mem[k])
+		}
+		for k := range s.del {
+			tempstore.drop(k)
+		}
+		s.ps = tempstore.ps
+		s.mem = tempstore.mem
+		s.del = tempstore.del
 	}
+	s.mut.Unlock()
 	return keys, err
 }
 

pkg/core/storage/memcached_store_test.go

@@ -177,3 +177,65 @@ func TestCachedSeek(t *testing.T) {
 func newMemCachedStoreForTesting(t *testing.T) Store {
 	return NewMemCachedStore(NewMemoryStore())
 }
+
+type BadBatch struct{}
+
+func (b BadBatch) Delete(k []byte) {}
+func (b BadBatch) Put(k, v []byte) {}
+
+type BadStore struct {
+	onPutBatch func()
+}
+
+func (b *BadStore) Batch() Batch {
+	return BadBatch{}
+}
+func (b *BadStore) Delete(k []byte) error {
+	return nil
+}
+func (b *BadStore) Get([]byte) ([]byte, error) {
+	return nil, ErrKeyNotFound
+}
+func (b *BadStore) Put(k, v []byte) error {
+	return nil
+}
+func (b *BadStore) PutBatch(Batch) error {
+	b.onPutBatch()
+	return ErrKeyNotFound
+}
+func (b *BadStore) Seek(k []byte, f func(k, v []byte)) {
+}
+func (b *BadStore) Close() error {
+	return nil
+}
+
+func TestMemCachedPersistFailing(t *testing.T) {
+	var (
+		bs BadStore
+		t1 = []byte("t1")
+		t2 = []byte("t2")
+		b1 = []byte("b1")
+	)
+	// cached Store
+	ts := NewMemCachedStore(&bs)
+	// Set a pair of keys.
+	require.NoError(t, ts.Put(t1, t1))
+	require.NoError(t, ts.Put(t2, t2))
+	// This will be called during Persist().
+	bs.onPutBatch = func() {
+		// Drop one, add one.
+		require.NoError(t, ts.Put(b1, b1))
+		require.NoError(t, ts.Delete(t1))
+	}
+	_, err := ts.Persist()
+	require.Error(t, err)
+	// PutBatch() failed in Persist, but we still should have proper state.
+	_, err = ts.Get(t1)
+	require.Error(t, err)
+	res, err := ts.Get(t2)
+	require.NoError(t, err)
+	require.Equal(t, t2, res)
+	res, err = ts.Get(b1)
+	require.NoError(t, err)
+	require.Equal(t, b1, res)
+}