neo-go/pkg/core/statesync_test.go
Anna Shaleva cd42b8b20c core: allow early Seek stop
This simple approach allows to improve the performance of
BoltDB and LevelDB in both terms of speed and allocations
for retrieving GasPerVote value from the storage.
MemoryPS's speed suffers a bit, but we don't use it for
production environment.

Part of #2322.

Benchmark results:

name                                                              old time/op    new time/op    delta
NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8           25.3µs ± 1%    26.4µs ± 9%   +4.41%  (p=0.043 n=10+9)
NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8          27.9µs ± 1%    30.1µs ±15%   +7.97%  (p=0.000 n=10+9)
NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8         55.1µs ± 1%    60.2µs ± 7%   +9.27%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8         353µs ± 2%     416µs ±13%  +17.88%  (p=0.000 n=8+8)
NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8           195µs ± 1%     216µs ± 7%  +10.42%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8          200µs ± 4%     214µs ± 9%   +6.99%  (p=0.002 n=9+8)
NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8         223µs ± 2%     247µs ± 9%  +10.60%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8        612µs ±23%     855µs ±52%  +39.60%  (p=0.001 n=9+10)
NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8         11.3ms ±53%    10.7ms ±50%     ~     (p=0.739 n=10+10)
NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8        12.0ms ±37%    10.4ms ±65%     ~     (p=0.853 n=10+10)
NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8       11.3ms ±40%    10.4ms ±49%     ~     (p=0.631 n=10+10)
NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8      3.80ms ±45%    3.69ms ±27%     ~     (p=0.931 n=6+5)
NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8          23.0µs ± 9%    22.6µs ± 4%     ~     (p=0.059 n=8+9)
NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8         25.9µs ± 5%    24.8µs ± 4%   -4.17%  (p=0.006 n=10+8)
NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8        42.7µs ±13%    38.9µs ± 1%   -8.85%  (p=0.000 n=9+8)
NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8       80.8µs ±12%    84.9µs ± 9%     ~     (p=0.114 n=8+9)
NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8         64.3µs ±16%    22.1µs ±23%  -65.64%  (p=0.000 n=10+10)
NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8        61.0µs ±34%    23.2µs ± 8%  -62.04%  (p=0.000 n=10+9)
NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8       62.2µs ±14%    25.7µs ±13%  -58.66%  (p=0.000 n=9+10)
NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8       359µs ±60%     325µs ±60%     ~     (p=0.739 n=10+10)
NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8         242µs ±21%      13µs ±28%  -94.49%  (p=0.000 n=10+8)
NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8        229µs ±23%      18µs ±70%  -92.02%  (p=0.000 n=10+9)
NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8       238µs ±28%     20µs ±109%  -91.38%  (p=0.000 n=10+9)
NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8      265µs ±20%      77µs ±62%  -71.04%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8         25.5µs ± 3%    24.7µs ± 7%     ~     (p=0.143 n=10+10)
NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8        27.4µs ± 2%    27.9µs ± 6%     ~     (p=0.280 n=10+10)
NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8       50.2µs ± 7%    47.4µs ±10%     ~     (p=0.156 n=9+10)
NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8      98.2µs ± 9%    94.6µs ±10%     ~     (p=0.218 n=10+10)
NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8        82.9µs ±13%    32.1µs ±22%  -61.30%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8       92.2µs ±11%    33.7µs ±12%  -63.42%  (p=0.000 n=10+9)
NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8      88.3µs ±22%    39.4µs ±14%  -55.36%  (p=0.000 n=10+9)
NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8      106µs ±18%      78µs ±24%  -26.20%  (p=0.000 n=9+10)
NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8        360µs ±24%      29µs ±53%  -91.91%  (p=0.000 n=10+9)
NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8       353µs ±16%      50µs ±70%  -85.72%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8      381µs ±20%     47µs ±111%  -87.64%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8     434µs ±19%     113µs ±41%  -74.04%  (p=0.000 n=10+10)

name                                                              old alloc/op   new alloc/op   delta
NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8           4.82kB ± 0%    4.26kB ± 1%  -11.62%  (p=0.000 n=10+9)
NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8          4.99kB ± 0%    4.41kB ± 1%  -11.56%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8         8.45kB ± 0%    7.87kB ± 0%   -6.88%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8        55.0kB ± 0%    54.5kB ± 0%   -0.81%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8          29.1kB ± 0%    21.7kB ± 2%  -25.56%  (p=0.000 n=9+9)
NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8         29.3kB ± 1%    21.8kB ± 2%  -25.74%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8        31.3kB ± 1%    23.6kB ± 1%  -24.50%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8       92.5kB ± 5%    84.7kB ± 3%   -8.50%  (p=0.000 n=10+9)
NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8          324kB ±29%     222kB ±44%  -31.33%  (p=0.007 n=10+10)
NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8         308kB ±32%     174kB ±14%  -43.56%  (p=0.000 n=10+8)
NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8        298kB ±23%     178kB ±36%  -40.26%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8       362kB ± 6%     248kB ± 6%  -31.54%  (p=0.004 n=6+5)
NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8          5.15kB ± 3%    4.64kB ± 2%   -9.92%  (p=0.000 n=10+9)
NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8         5.36kB ± 1%    4.75kB ± 5%  -11.42%  (p=0.000 n=10+10)
NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8        8.15kB ± 4%    7.53kB ± 1%   -7.62%  (p=0.000 n=10+9)
NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8       33.2kB ± 5%    33.2kB ± 7%     ~     (p=0.829 n=8+10)
NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8         20.1kB ± 7%     5.8kB ±13%  -70.90%  (p=0.000 n=10+10)
NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8        19.8kB ±14%     6.2kB ± 5%  -68.87%  (p=0.000 n=10+9)
NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8       21.7kB ± 6%     8.0kB ± 7%  -63.20%  (p=0.000 n=9+10)
NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8      98.5kB ±44%    81.8kB ±48%     ~     (p=0.143 n=10+10)
NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8         130kB ± 4%       4kB ± 9%  -96.69%  (p=0.000 n=10+10)
NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8        131kB ± 4%       5kB ±21%  -96.48%  (p=0.000 n=9+9)
NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8       132kB ± 4%       6kB ±10%  -95.39%  (p=0.000 n=10+8)
NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8      151kB ± 4%      26kB ±10%  -82.46%  (p=0.000 n=9+9)
NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8         5.92kB ± 3%    5.32kB ± 2%  -10.01%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8        6.09kB ± 2%    5.48kB ± 2%  -10.00%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8       9.61kB ± 1%    9.00kB ± 0%   -6.29%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8      33.4kB ± 7%    32.2kB ± 5%   -3.60%  (p=0.037 n=10+10)
NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8        22.3kB ±10%     9.0kB ±16%  -59.78%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8       23.6kB ± 6%     8.5kB ±20%  -63.76%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8      24.2kB ± 9%    11.5kB ± 4%  -52.34%  (p=0.000 n=10+8)
NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8     44.2kB ± 6%    30.8kB ± 9%  -30.24%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8        144kB ± 4%      10kB ±24%  -93.39%  (p=0.000 n=9+10)
NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8       146kB ± 1%      11kB ±37%  -92.14%  (p=0.000 n=7+10)
NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8      149kB ± 3%      11kB ±12%  -92.28%  (p=0.000 n=10+9)
NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8     171kB ± 4%      34kB ±12%  -80.00%  (p=0.000 n=10+10)

name                                                              old allocs/op  new allocs/op  delta
NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8             95.0 ± 0%      74.0 ± 0%  -22.11%  (p=0.001 n=8+9)
NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8             100 ± 0%        78 ± 1%  -21.70%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8            153 ± 0%       131 ± 2%  -14.25%  (p=0.000 n=6+10)
NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8           799 ± 2%       797 ± 4%     ~     (p=0.956 n=10+10)
NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8             438 ± 6%       167 ± 0%  -61.86%  (p=0.000 n=10+9)
NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8            446 ± 5%       172 ± 0%  -61.38%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8           506 ± 4%       232 ± 1%  -54.21%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8        1.31k ± 5%     0.97k ± 4%  -26.20%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8          5.06k ± 1%     1.09k ± 2%  -78.53%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8         5.02k ± 3%     1.08k ± 0%  -78.45%  (p=0.000 n=10+8)
NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8        5.09k ± 3%     1.15k ± 2%  -77.48%  (p=0.000 n=10+10)
NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8       5.83k ± 1%     1.87k ± 3%  -68.02%  (p=0.004 n=6+5)
NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8             103 ± 2%        82 ± 1%  -20.83%  (p=0.000 n=10+10)
NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8            107 ± 0%        86 ± 0%  -19.63%  (p=0.000 n=8+8)
NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8           164 ± 1%       139 ± 0%  -15.45%  (p=0.000 n=10+9)
NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8          820 ± 1%       789 ± 1%   -3.70%  (p=0.000 n=9+10)
NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8            475 ± 0%        94 ± 3%  -80.15%  (p=0.000 n=10+9)
NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8           481 ± 0%       100 ± 2%  -79.26%  (p=0.000 n=9+9)
NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8          549 ± 0%       161 ± 2%  -70.69%  (p=0.000 n=10+10)
NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8       1.61k ±19%     1.19k ±25%  -26.05%  (p=0.000 n=10+10)
NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8         4.12k ± 0%     0.08k ± 2%  -98.02%  (p=0.000 n=10+10)
NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8        4.14k ± 0%     0.09k ± 3%  -97.90%  (p=0.000 n=9+9)
NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8       4.19k ± 0%     0.15k ± 3%  -96.52%  (p=0.000 n=9+9)
NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8      4.82k ± 1%     0.74k ± 1%  -84.58%  (p=0.000 n=10+9)
NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8            112 ± 4%        90 ± 3%  -19.45%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8           116 ± 2%        95 ± 2%  -17.90%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8          170 ± 3%       148 ± 3%  -12.99%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8         800 ± 2%       772 ± 2%   -3.50%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8           480 ± 3%       118 ± 3%  -75.32%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8          479 ± 2%       123 ± 3%  -74.33%  (p=0.000 n=10+9)
NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8         542 ± 1%       183 ± 3%  -66.34%  (p=0.000 n=10+9)
NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8      1.19k ± 1%     0.79k ± 1%  -33.41%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8        4.21k ± 1%     0.13k ±21%  -96.83%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8       4.23k ± 1%     0.15k ±17%  -96.48%  (p=0.000 n=10+10)
NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8      4.27k ± 0%     0.19k ± 6%  -95.51%  (p=0.000 n=10+9)
NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8     4.89k ± 1%     0.79k ± 2%  -83.80%  (p=0.000 n=10+10)
2022-01-19 20:54:35 +03:00

462 lines
17 KiB
Go

package core
import (
"testing"
"time"
"github.com/nspcc-dev/neo-go/pkg/config"
"github.com/nspcc-dev/neo-go/pkg/core/block"
"github.com/nspcc-dev/neo-go/pkg/core/mpt"
"github.com/nspcc-dev/neo-go/pkg/core/storage"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/util/slice"
"github.com/stretchr/testify/require"
)
func TestStateSyncModule_Init(t *testing.T) {
var (
stateSyncInterval = 2
maxTraceable uint32 = 3
)
spoutCfg := func(c *config.Config) {
c.ProtocolConfiguration.StateRootInHeader = true
c.ProtocolConfiguration.P2PStateExchangeExtensions = true
c.ProtocolConfiguration.StateSyncInterval = stateSyncInterval
c.ProtocolConfiguration.MaxTraceableBlocks = maxTraceable
}
bcSpout := newTestChainWithCustomCfg(t, spoutCfg)
for i := 0; i <= 2*stateSyncInterval+int(maxTraceable)+1; i++ {
require.NoError(t, bcSpout.AddBlock(bcSpout.newBlock()))
}
boltCfg := func(c *config.Config) {
spoutCfg(c)
c.ProtocolConfiguration.KeepOnlyLatestState = true
c.ProtocolConfiguration.RemoveUntraceableBlocks = true
}
t.Run("error: module disabled by config", func(t *testing.T) {
bcBolt := newTestChainWithCustomCfg(t, func(c *config.Config) {
boltCfg(c)
c.ProtocolConfiguration.RemoveUntraceableBlocks = false
})
module := bcBolt.GetStateSyncModule()
require.Error(t, module.Init(bcSpout.BlockHeight())) // module inactive (non-archival node)
})
t.Run("inactive: spout chain is too low to start state sync process", func(t *testing.T) {
bcBolt := newTestChainWithCustomCfg(t, boltCfg)
module := bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(uint32(2*stateSyncInterval-1)))
require.False(t, module.IsActive())
})
t.Run("inactive: bolt chain height is close enough to spout chain height", func(t *testing.T) {
bcBolt := newTestChainWithCustomCfg(t, boltCfg)
for i := 1; i < int(bcSpout.BlockHeight())-stateSyncInterval; i++ {
b, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(i))
require.NoError(t, err)
require.NoError(t, bcBolt.AddBlock(b))
}
module := bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(bcSpout.BlockHeight()))
require.False(t, module.IsActive())
})
t.Run("error: bolt chain is too low to start state sync process", func(t *testing.T) {
bcBolt := newTestChainWithCustomCfg(t, boltCfg)
require.NoError(t, bcBolt.AddBlock(bcBolt.newBlock()))
module := bcBolt.GetStateSyncModule()
require.Error(t, module.Init(uint32(3*stateSyncInterval)))
})
t.Run("initialized: no previous state sync point", func(t *testing.T) {
bcBolt := newTestChainWithCustomCfg(t, boltCfg)
module := bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(bcSpout.BlockHeight()))
require.True(t, module.IsActive())
require.True(t, module.IsInitialized())
require.True(t, module.NeedHeaders())
require.False(t, module.NeedMPTNodes())
})
t.Run("error: outdated state sync point in the storage", func(t *testing.T) {
bcBolt := newTestChainWithCustomCfg(t, boltCfg)
module := bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(bcSpout.BlockHeight()))
module = bcBolt.GetStateSyncModule()
require.Error(t, module.Init(bcSpout.BlockHeight()+2*uint32(stateSyncInterval)))
})
t.Run("initialized: valid previous state sync point in the storage", func(t *testing.T) {
bcBolt := newTestChainWithCustomCfg(t, boltCfg)
module := bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(bcSpout.BlockHeight()))
module = bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(bcSpout.BlockHeight()))
require.True(t, module.IsActive())
require.True(t, module.IsInitialized())
require.True(t, module.NeedHeaders())
require.False(t, module.NeedMPTNodes())
})
t.Run("initialization from headers/blocks/mpt synced stages", func(t *testing.T) {
bcBolt := newTestChainWithCustomCfg(t, boltCfg)
module := bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(bcSpout.BlockHeight()))
// firstly, fetch all headers to create proper DB state (where headers are in sync)
stateSyncPoint := (int(bcSpout.BlockHeight()) / stateSyncInterval) * stateSyncInterval
var expectedHeader *block.Header
for i := 1; i <= int(bcSpout.HeaderHeight()); i++ {
header, err := bcSpout.GetHeader(bcSpout.GetHeaderHash(i))
require.NoError(t, err)
require.NoError(t, module.AddHeaders(header))
if i == stateSyncPoint+1 {
expectedHeader = header
}
}
require.True(t, module.IsActive())
require.True(t, module.IsInitialized())
require.False(t, module.NeedHeaders())
require.True(t, module.NeedMPTNodes())
// then create new statesync module with the same DB and check that state is proper
// (headers are in sync)
module = bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(bcSpout.BlockHeight()))
require.True(t, module.IsActive())
require.True(t, module.IsInitialized())
require.False(t, module.NeedHeaders())
require.True(t, module.NeedMPTNodes())
unknownNodes := module.GetUnknownMPTNodesBatch(2)
require.Equal(t, 1, len(unknownNodes))
require.Equal(t, expectedHeader.PrevStateRoot, unknownNodes[0])
// add several blocks to create DB state where blocks are not in sync yet, but it's not a genesis.
for i := stateSyncPoint - int(maxTraceable) + 1; i <= stateSyncPoint-stateSyncInterval-1; i++ {
block, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(i))
require.NoError(t, err)
require.NoError(t, module.AddBlock(block))
}
require.True(t, module.IsActive())
require.True(t, module.IsInitialized())
require.False(t, module.NeedHeaders())
require.True(t, module.NeedMPTNodes())
require.Equal(t, uint32(stateSyncPoint-stateSyncInterval-1), module.BlockHeight())
// then create new statesync module with the same DB and check that state is proper
// (blocks are not in sync yet)
module = bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(bcSpout.BlockHeight()))
require.True(t, module.IsActive())
require.True(t, module.IsInitialized())
require.False(t, module.NeedHeaders())
require.True(t, module.NeedMPTNodes())
unknownNodes = module.GetUnknownMPTNodesBatch(2)
require.Equal(t, 1, len(unknownNodes))
require.Equal(t, expectedHeader.PrevStateRoot, unknownNodes[0])
require.Equal(t, uint32(stateSyncPoint-stateSyncInterval-1), module.BlockHeight())
// add rest of blocks to create DB state where blocks are in sync
for i := stateSyncPoint - stateSyncInterval; i <= stateSyncPoint; i++ {
block, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(i))
require.NoError(t, err)
require.NoError(t, module.AddBlock(block))
}
require.True(t, module.IsActive())
require.True(t, module.IsInitialized())
require.False(t, module.NeedHeaders())
require.True(t, module.NeedMPTNodes())
lastBlock, err := bcBolt.GetBlock(expectedHeader.PrevHash)
require.NoError(t, err)
require.Equal(t, uint32(stateSyncPoint), lastBlock.Index)
require.Equal(t, uint32(stateSyncPoint), module.BlockHeight())
// then create new statesync module with the same DB and check that state is proper
// (headers and blocks are in sync)
module = bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(bcSpout.BlockHeight()))
require.True(t, module.IsActive())
require.True(t, module.IsInitialized())
require.False(t, module.NeedHeaders())
require.True(t, module.NeedMPTNodes())
unknownNodes = module.GetUnknownMPTNodesBatch(2)
require.Equal(t, 1, len(unknownNodes))
require.Equal(t, expectedHeader.PrevStateRoot, unknownNodes[0])
require.Equal(t, uint32(stateSyncPoint), module.BlockHeight())
// add a few MPT nodes to create DB state where some of MPT nodes are missing
count := 5
for {
unknownHashes := module.GetUnknownMPTNodesBatch(1) // restore nodes one-by-one
if len(unknownHashes) == 0 {
break
}
err := bcSpout.GetStateSyncModule().Traverse(unknownHashes[0], func(node mpt.Node, nodeBytes []byte) bool {
require.NoError(t, module.AddMPTNodes([][]byte{nodeBytes}))
return true // add nodes one-by-one
})
require.NoError(t, err)
count--
if count < 0 {
break
}
}
// then create new statesync module with the same DB and check that state is proper
// (headers and blocks are in sync, mpt is not yet synced)
module = bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(bcSpout.BlockHeight()))
require.True(t, module.IsActive())
require.True(t, module.IsInitialized())
require.False(t, module.NeedHeaders())
require.True(t, module.NeedMPTNodes())
unknownNodes = module.GetUnknownMPTNodesBatch(100)
require.True(t, len(unknownNodes) > 0)
require.NotContains(t, unknownNodes, expectedHeader.PrevStateRoot)
require.Equal(t, uint32(stateSyncPoint), module.BlockHeight())
// add the rest of MPT nodes and jump to state
alreadyRequested := make(map[util.Uint256]struct{})
for {
unknownHashes := module.GetUnknownMPTNodesBatch(1) // restore nodes one-by-one
if len(unknownHashes) == 0 {
break
}
if _, ok := alreadyRequested[unknownHashes[0]]; ok {
t.Fatal("bug: node was requested twice")
}
alreadyRequested[unknownHashes[0]] = struct{}{}
var callbackCalled bool
err := bcSpout.GetStateSyncModule().Traverse(unknownHashes[0], func(node mpt.Node, nodeBytes []byte) bool {
require.NoError(t, module.AddMPTNodes([][]byte{slice.Copy(nodeBytes)}))
callbackCalled = true
return true // add nodes one-by-one
})
require.NoError(t, err)
require.True(t, callbackCalled)
}
// check that module is inactive and statejump is completed
require.False(t, module.IsActive())
require.False(t, module.NeedHeaders())
require.False(t, module.NeedMPTNodes())
unknownNodes = module.GetUnknownMPTNodesBatch(1)
require.True(t, len(unknownNodes) == 0)
require.Equal(t, uint32(stateSyncPoint), module.BlockHeight())
require.Equal(t, uint32(stateSyncPoint), bcBolt.BlockHeight())
// create new module from completed state: the module should recognise that state sync is completed
module = bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(bcSpout.BlockHeight()))
require.False(t, module.IsActive())
require.False(t, module.NeedHeaders())
require.False(t, module.NeedMPTNodes())
unknownNodes = module.GetUnknownMPTNodesBatch(1)
require.True(t, len(unknownNodes) == 0)
require.Equal(t, uint32(stateSyncPoint), module.BlockHeight())
require.Equal(t, uint32(stateSyncPoint), bcBolt.BlockHeight())
// add one more block to the restored chain and start new module: the module should recognise state sync is completed
// and regular blocks processing was started
require.NoError(t, bcBolt.AddBlock(bcBolt.newBlock()))
module = bcBolt.GetStateSyncModule()
require.NoError(t, module.Init(bcSpout.BlockHeight()))
require.False(t, module.IsActive())
require.False(t, module.NeedHeaders())
require.False(t, module.NeedMPTNodes())
unknownNodes = module.GetUnknownMPTNodesBatch(1)
require.True(t, len(unknownNodes) == 0)
require.Equal(t, uint32(stateSyncPoint)+1, module.BlockHeight())
require.Equal(t, uint32(stateSyncPoint)+1, bcBolt.BlockHeight())
})
}
func TestStateSyncModule_RestoreBasicChain(t *testing.T) {
var (
stateSyncInterval = 4
maxTraceable uint32 = 6
stateSyncPoint = 16
)
spoutCfg := func(c *config.Config) {
c.ProtocolConfiguration.StateRootInHeader = true
c.ProtocolConfiguration.P2PStateExchangeExtensions = true
c.ProtocolConfiguration.StateSyncInterval = stateSyncInterval
c.ProtocolConfiguration.MaxTraceableBlocks = maxTraceable
}
bcSpout := newTestChainWithCustomCfg(t, spoutCfg)
initBasicChain(t, bcSpout)
// make spout chain higher that latest state sync point
require.NoError(t, bcSpout.AddBlock(bcSpout.newBlock()))
require.NoError(t, bcSpout.AddBlock(bcSpout.newBlock()))
require.Equal(t, uint32(stateSyncPoint+2), bcSpout.BlockHeight())
boltCfg := func(c *config.Config) {
spoutCfg(c)
c.ProtocolConfiguration.KeepOnlyLatestState = true
c.ProtocolConfiguration.RemoveUntraceableBlocks = true
}
bcBoltStore := memoryStore{storage.NewMemoryStore()}
bcBolt := initTestChain(t, bcBoltStore, boltCfg)
go bcBolt.Run()
module := bcBolt.GetStateSyncModule()
t.Run("error: add headers before initialisation", func(t *testing.T) {
h, err := bcSpout.GetHeader(bcSpout.GetHeaderHash(1))
require.NoError(t, err)
require.Error(t, module.AddHeaders(h))
})
t.Run("no error: add blocks before initialisation", func(t *testing.T) {
b, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(1))
require.NoError(t, err)
require.NoError(t, module.AddBlock(b))
})
t.Run("error: add MPT nodes without initialisation", func(t *testing.T) {
require.Error(t, module.AddMPTNodes([][]byte{}))
})
require.NoError(t, module.Init(bcSpout.BlockHeight()))
require.True(t, module.IsActive())
require.True(t, module.IsInitialized())
require.True(t, module.NeedHeaders())
require.False(t, module.NeedMPTNodes())
// add headers to module
headers := make([]*block.Header, 0, bcSpout.HeaderHeight())
for i := uint32(1); i <= bcSpout.HeaderHeight(); i++ {
h, err := bcSpout.GetHeader(bcSpout.GetHeaderHash(int(i)))
require.NoError(t, err)
headers = append(headers, h)
}
require.NoError(t, module.AddHeaders(headers...))
require.True(t, module.IsActive())
require.True(t, module.IsInitialized())
require.False(t, module.NeedHeaders())
require.True(t, module.NeedMPTNodes())
require.Equal(t, bcSpout.HeaderHeight(), bcBolt.HeaderHeight())
// add blocks
t.Run("error: unexpected block index", func(t *testing.T) {
b, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(stateSyncPoint - int(maxTraceable)))
require.NoError(t, err)
require.Error(t, module.AddBlock(b))
})
t.Run("error: missing state root in block header", func(t *testing.T) {
b := &block.Block{
Header: block.Header{
Index: uint32(stateSyncPoint) - maxTraceable + 1,
StateRootEnabled: false,
},
}
require.Error(t, module.AddBlock(b))
})
t.Run("error: invalid block merkle root", func(t *testing.T) {
b := &block.Block{
Header: block.Header{
Index: uint32(stateSyncPoint) - maxTraceable + 1,
StateRootEnabled: true,
MerkleRoot: util.Uint256{1, 2, 3},
},
}
require.Error(t, module.AddBlock(b))
})
for i := stateSyncPoint - int(maxTraceable) + 1; i <= stateSyncPoint; i++ {
b, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(i))
require.NoError(t, err)
require.NoError(t, module.AddBlock(b))
}
require.True(t, module.IsActive())
require.True(t, module.IsInitialized())
require.False(t, module.NeedHeaders())
require.True(t, module.NeedMPTNodes())
require.Equal(t, uint32(stateSyncPoint), module.BlockHeight())
// add MPT nodes in batches
h, err := bcSpout.GetHeader(bcSpout.GetHeaderHash(stateSyncPoint + 1))
require.NoError(t, err)
unknownHashes := module.GetUnknownMPTNodesBatch(100)
require.Equal(t, 1, len(unknownHashes))
require.Equal(t, h.PrevStateRoot, unknownHashes[0])
nodesMap := make(map[util.Uint256][]byte)
err = bcSpout.GetStateSyncModule().Traverse(h.PrevStateRoot, func(n mpt.Node, nodeBytes []byte) bool {
nodesMap[n.Hash()] = nodeBytes
return false
})
require.NoError(t, err)
for {
need := module.GetUnknownMPTNodesBatch(10)
if len(need) == 0 {
break
}
add := make([][]byte, len(need))
for i, h := range need {
nodeBytes, ok := nodesMap[h]
if !ok {
t.Fatal("unknown or restored node requested")
}
add[i] = nodeBytes
delete(nodesMap, h)
}
require.NoError(t, module.AddMPTNodes(add))
}
require.False(t, module.IsActive())
require.False(t, module.NeedHeaders())
require.False(t, module.NeedMPTNodes())
unknownNodes := module.GetUnknownMPTNodesBatch(1)
require.True(t, len(unknownNodes) == 0)
require.Equal(t, uint32(stateSyncPoint), module.BlockHeight())
require.Equal(t, uint32(stateSyncPoint), bcBolt.BlockHeight())
// add missing blocks to bcBolt: should be ok, because state is synced
for i := stateSyncPoint + 1; i <= int(bcSpout.BlockHeight()); i++ {
b, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(i))
require.NoError(t, err)
require.NoError(t, bcBolt.AddBlock(b))
}
require.Equal(t, bcSpout.BlockHeight(), bcBolt.BlockHeight())
// compare storage states
fetchStorage := func(bc *Blockchain) []storage.KeyValue {
var kv []storage.KeyValue
bc.dao.Store.Seek(storage.SeekRange{Prefix: bc.dao.Version.StoragePrefix.Bytes()}, func(k, v []byte) bool {
key := slice.Copy(k)
value := slice.Copy(v)
if key[0] == byte(storage.STTempStorage) {
key[0] = byte(storage.STStorage)
}
kv = append(kv, storage.KeyValue{
Key: key,
Value: value,
})
return true
})
return kv
}
expected := fetchStorage(bcSpout)
actual := fetchStorage(bcBolt)
require.ElementsMatch(t, expected, actual)
// no temp items should be left
require.Eventually(t, func() bool {
var haveItems bool
bcBolt.dao.Store.Seek(storage.SeekRange{Prefix: storage.STStorage.Bytes()}, func(_, _ []byte) bool {
haveItems = true
return false
})
return !haveItems
}, time.Second*5, time.Millisecond*100)
bcBolt.Close()
// Check restoring with new prefix.
bcBolt = initTestChain(t, bcBoltStore, boltCfg)
go bcBolt.Run()
defer bcBolt.Close()
require.Equal(t, storage.STTempStorage, bcBolt.dao.Version.StoragePrefix)
require.Equal(t, storage.STTempStorage, bcBolt.persistent.Version.StoragePrefix)
}