neo-go/pkg/core/storage/store.go

176 lines
6 KiB
Go
Raw Normal View History

package storage
import (
"errors"
"fmt"
"github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig"
"github.com/nspcc-dev/neo-go/pkg/core/storage/dboper"
"github.com/syndtr/goleveldb/leveldb/util"
)
// KeyPrefix constants.
const (
DataExecutable KeyPrefix = 0x01
// DataMPT is used for MPT node entries identified by Uint256.
DataMPT KeyPrefix = 0x03
// DataMPTAux is used to store additional MPT data like height-root
// mappings and local/validated heights.
DataMPTAux KeyPrefix = 0x04
STStorage KeyPrefix = 0x70
// STTempStorage is used to store contract storage items during state sync process
// in order not to mess up the previous state which has its own items stored by
// STStorage prefix. Once state exchange process is completed, all items with
// STStorage prefix will be replaced with STTempStorage-prefixed ones.
STTempStorage KeyPrefix = 0x71
STNEP11Transfers KeyPrefix = 0x72
STNEP17Transfers KeyPrefix = 0x73
STTokenTransferInfo KeyPrefix = 0x74
IXHeaderHashList KeyPrefix = 0x80
SYSCurrentBlock KeyPrefix = 0xc0
SYSCurrentHeader KeyPrefix = 0xc1
SYSStateSyncCurrentBlockHeight KeyPrefix = 0xc2
SYSStateSyncPoint KeyPrefix = 0xc3
2022-10-20 10:59:19 +00:00
// SYSStateChangeStage is used to store the phase of a state changing process
// which is one of the state jump or state reset. Its value is one byte containing
// state reset / state jump stages bits (first seven bits are reserved for that)
// and the last bit reserved for the state reset process marker (set to 1 on
// unfinished state reset and to 0 on unfinished state jump).
SYSStateChangeStage KeyPrefix = 0xc4
SYSVersion KeyPrefix = 0xf0
)
// Executable subtypes.
const (
ExecBlock byte = 1
ExecTransaction byte = 2
)
// SeekRange represents options for Store.Seek operation.
type SeekRange struct {
// Prefix denotes the Seek's lookup key.
// If used directly with MemCachedStore's or MemoryStore's Seek, SeekAsync or
// SeekGC empty Prefix is not supported due to internal MemoryStore cache
// architecture; otherwise empty prefix can be used safely.
Prefix []byte
core: allow early Seek stop This simple approach allows to improve the performance of BoltDB and LevelDB in both terms of speed and allocations for retrieving GasPerVote value from the storage. MemoryPS's speed suffers a bit, but we don't use it for production environment. Part of #2322. Benchmark results: name old time/op new time/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 25.3µs ± 1% 26.4µs ± 9% +4.41% (p=0.043 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 27.9µs ± 1% 30.1µs ±15% +7.97% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 55.1µs ± 1% 60.2µs ± 7% +9.27% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 353µs ± 2% 416µs ±13% +17.88% (p=0.000 n=8+8) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 195µs ± 1% 216µs ± 7% +10.42% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 200µs ± 4% 214µs ± 9% +6.99% (p=0.002 n=9+8) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 223µs ± 2% 247µs ± 9% +10.60% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 612µs ±23% 855µs ±52% +39.60% (p=0.001 n=9+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 11.3ms ±53% 10.7ms ±50% ~ (p=0.739 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 12.0ms ±37% 10.4ms ±65% ~ (p=0.853 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 11.3ms ±40% 10.4ms ±49% ~ (p=0.631 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 3.80ms ±45% 3.69ms ±27% ~ (p=0.931 n=6+5) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 23.0µs ± 9% 22.6µs ± 4% ~ (p=0.059 n=8+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 25.9µs ± 5% 24.8µs ± 4% -4.17% (p=0.006 n=10+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 42.7µs ±13% 38.9µs ± 1% -8.85% (p=0.000 n=9+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 80.8µs ±12% 84.9µs ± 9% ~ (p=0.114 n=8+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 64.3µs ±16% 22.1µs ±23% -65.64% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 61.0µs ±34% 23.2µs ± 8% -62.04% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 62.2µs ±14% 25.7µs ±13% -58.66% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 359µs ±60% 325µs ±60% ~ (p=0.739 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 242µs ±21% 13µs ±28% -94.49% (p=0.000 n=10+8) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 229µs ±23% 18µs ±70% -92.02% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 238µs ±28% 20µs ±109% -91.38% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 265µs ±20% 77µs ±62% -71.04% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 25.5µs ± 3% 24.7µs ± 7% ~ (p=0.143 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 27.4µs ± 2% 27.9µs ± 6% ~ (p=0.280 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 50.2µs ± 7% 47.4µs ±10% ~ (p=0.156 n=9+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 98.2µs ± 9% 94.6µs ±10% ~ (p=0.218 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 82.9µs ±13% 32.1µs ±22% -61.30% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 92.2µs ±11% 33.7µs ±12% -63.42% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 88.3µs ±22% 39.4µs ±14% -55.36% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 106µs ±18% 78µs ±24% -26.20% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 360µs ±24% 29µs ±53% -91.91% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 353µs ±16% 50µs ±70% -85.72% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 381µs ±20% 47µs ±111% -87.64% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 434µs ±19% 113µs ±41% -74.04% (p=0.000 n=10+10) name old alloc/op new alloc/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 4.82kB ± 0% 4.26kB ± 1% -11.62% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 4.99kB ± 0% 4.41kB ± 1% -11.56% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 8.45kB ± 0% 7.87kB ± 0% -6.88% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 55.0kB ± 0% 54.5kB ± 0% -0.81% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 29.1kB ± 0% 21.7kB ± 2% -25.56% (p=0.000 n=9+9) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 29.3kB ± 1% 21.8kB ± 2% -25.74% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 31.3kB ± 1% 23.6kB ± 1% -24.50% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 92.5kB ± 5% 84.7kB ± 3% -8.50% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 324kB ±29% 222kB ±44% -31.33% (p=0.007 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 308kB ±32% 174kB ±14% -43.56% (p=0.000 n=10+8) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 298kB ±23% 178kB ±36% -40.26% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 362kB ± 6% 248kB ± 6% -31.54% (p=0.004 n=6+5) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 5.15kB ± 3% 4.64kB ± 2% -9.92% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 5.36kB ± 1% 4.75kB ± 5% -11.42% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 8.15kB ± 4% 7.53kB ± 1% -7.62% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 33.2kB ± 5% 33.2kB ± 7% ~ (p=0.829 n=8+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 20.1kB ± 7% 5.8kB ±13% -70.90% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 19.8kB ±14% 6.2kB ± 5% -68.87% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 21.7kB ± 6% 8.0kB ± 7% -63.20% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 98.5kB ±44% 81.8kB ±48% ~ (p=0.143 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 130kB ± 4% 4kB ± 9% -96.69% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 131kB ± 4% 5kB ±21% -96.48% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 132kB ± 4% 6kB ±10% -95.39% (p=0.000 n=10+8) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 151kB ± 4% 26kB ±10% -82.46% (p=0.000 n=9+9) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 5.92kB ± 3% 5.32kB ± 2% -10.01% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 6.09kB ± 2% 5.48kB ± 2% -10.00% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 9.61kB ± 1% 9.00kB ± 0% -6.29% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 33.4kB ± 7% 32.2kB ± 5% -3.60% (p=0.037 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 22.3kB ±10% 9.0kB ±16% -59.78% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 23.6kB ± 6% 8.5kB ±20% -63.76% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 24.2kB ± 9% 11.5kB ± 4% -52.34% (p=0.000 n=10+8) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 44.2kB ± 6% 30.8kB ± 9% -30.24% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 144kB ± 4% 10kB ±24% -93.39% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 146kB ± 1% 11kB ±37% -92.14% (p=0.000 n=7+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 149kB ± 3% 11kB ±12% -92.28% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 171kB ± 4% 34kB ±12% -80.00% (p=0.000 n=10+10) name old allocs/op new allocs/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 95.0 ± 0% 74.0 ± 0% -22.11% (p=0.001 n=8+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 100 ± 0% 78 ± 1% -21.70% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 153 ± 0% 131 ± 2% -14.25% (p=0.000 n=6+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 799 ± 2% 797 ± 4% ~ (p=0.956 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 438 ± 6% 167 ± 0% -61.86% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 446 ± 5% 172 ± 0% -61.38% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 506 ± 4% 232 ± 1% -54.21% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 1.31k ± 5% 0.97k ± 4% -26.20% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 5.06k ± 1% 1.09k ± 2% -78.53% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 5.02k ± 3% 1.08k ± 0% -78.45% (p=0.000 n=10+8) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 5.09k ± 3% 1.15k ± 2% -77.48% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 5.83k ± 1% 1.87k ± 3% -68.02% (p=0.004 n=6+5) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 103 ± 2% 82 ± 1% -20.83% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 107 ± 0% 86 ± 0% -19.63% (p=0.000 n=8+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 164 ± 1% 139 ± 0% -15.45% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 820 ± 1% 789 ± 1% -3.70% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 475 ± 0% 94 ± 3% -80.15% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 481 ± 0% 100 ± 2% -79.26% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 549 ± 0% 161 ± 2% -70.69% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 1.61k ±19% 1.19k ±25% -26.05% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 4.12k ± 0% 0.08k ± 2% -98.02% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 4.14k ± 0% 0.09k ± 3% -97.90% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 4.19k ± 0% 0.15k ± 3% -96.52% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 4.82k ± 1% 0.74k ± 1% -84.58% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 112 ± 4% 90 ± 3% -19.45% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 116 ± 2% 95 ± 2% -17.90% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 170 ± 3% 148 ± 3% -12.99% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 800 ± 2% 772 ± 2% -3.50% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 480 ± 3% 118 ± 3% -75.32% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 479 ± 2% 123 ± 3% -74.33% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 542 ± 1% 183 ± 3% -66.34% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 1.19k ± 1% 0.79k ± 1% -33.41% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 4.21k ± 1% 0.13k ±21% -96.83% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 4.23k ± 1% 0.15k ±17% -96.48% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 4.27k ± 0% 0.19k ± 6% -95.51% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 4.89k ± 1% 0.79k ± 2% -83.80% (p=0.000 n=10+10)
2022-01-17 17:41:51 +00:00
// Start denotes value appended to the Prefix to start Seek from.
// Seeking starting from some key includes this key to the result;
// if no matching key was found then next suitable key is picked up.
// Start may be empty. Empty Start means seeking through all keys in
// the DB with matching Prefix.
// Empty Prefix and empty Start can be combined, which means seeking
// through all keys in the DB, but see the Prefix's comment.
Start []byte
2021-12-28 13:01:44 +00:00
// Backwards denotes whether Seek direction should be reversed, i.e.
// whether seeking should be performed in a descending way.
// Backwards can be safely combined with Prefix and Start.
Backwards bool
// SearchDepth is the depth of Seek operation, denotes the number of cached
// DAO layers to perform search. Use 1 to fetch the latest changes from upper
// in-memory layer of cached DAO. Default 0 value denotes searching through
// the whole set of cached layers.
SearchDepth int
}
// ErrKeyNotFound is an error returned by Store implementations
// when a certain key is not found.
var ErrKeyNotFound = errors.New("key not found")
type (
// Store is the underlying KV backend for the blockchain data, it's
// not intended to be used directly, you wrap it with some memory cache
// layer most of the time.
Store interface {
Get([]byte) ([]byte, error)
storage: introduce PutChangeSet and use it for Persist We're using batches in wrong way during persist, we already have all changes accumulated in two maps and then we move them to batch and then this is applied. For some DBs like BoltDB this batch is just another MemoryStore, so we essentially just shuffle the changeset from one map to another, for others like LevelDB batch is just a serialized set of KV pairs, it doesn't help much on subsequent PutBatch, we just duplicate the changeset again. So introduce PutChangeSet that allows to take two maps with sets and deletes directly. It also allows to simplify MemCachedStore logic. neo-bench for single node with 10 workers, LevelDB: Reference: RPS 30189.132 30556.448 30390.482 ≈ 30379 ± 0.61% TPS 29427.344 29418.687 29434.273 ≈ 29427 ± 0.03% CPU % 33.304 27.179 33.860 ≈ 31.45 ± 11.79% Mem MB 800.677 798.389 715.042 ≈ 771 ± 6.33% Patched: RPS 30264.326 30386.364 30166.231 ≈ 30272 ± 0.36% ⇅ TPS 29444.673 29407.440 29452.478 ≈ 29435 ± 0.08% ⇅ CPU % 34.012 32.597 33.467 ≈ 33.36 ± 2.14% ⇅ Mem MB 549.126 523.656 517.684 ≈ 530 ± 3.15% ↓ 31.26% BoltDB: Reference: RPS 31937.647 31551.684 31850.408 ≈ 31780 ± 0.64% TPS 31292.049 30368.368 31307.724 ≈ 30989 ± 1.74% CPU % 33.792 22.339 35.887 ≈ 30.67 ± 23.78% Mem MB 1271.687 1254.472 1215.639 ≈ 1247 ± 2.30% Patched: RPS 31746.818 30859.485 31689.761 ≈ 31432 ± 1.58% ⇅ TPS 31271.499 30340.726 30342.568 ≈ 30652 ± 1.75% ⇅ CPU % 34.611 34.414 31.553 ≈ 33.53 ± 5.11% ⇅ Mem MB 1262.960 1231.389 1335.569 ≈ 1277 ± 4.18% ⇅
2021-08-12 10:35:09 +00:00
// PutChangeSet allows to push prepared changeset to the Store.
PutChangeSet(puts map[string][]byte, stor map[string][]byte) error
// Seek can guarantee that provided key (k) and value (v) are the only valid until the next call to f.
core: allow early Seek stop This simple approach allows to improve the performance of BoltDB and LevelDB in both terms of speed and allocations for retrieving GasPerVote value from the storage. MemoryPS's speed suffers a bit, but we don't use it for production environment. Part of #2322. Benchmark results: name old time/op new time/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 25.3µs ± 1% 26.4µs ± 9% +4.41% (p=0.043 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 27.9µs ± 1% 30.1µs ±15% +7.97% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 55.1µs ± 1% 60.2µs ± 7% +9.27% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 353µs ± 2% 416µs ±13% +17.88% (p=0.000 n=8+8) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 195µs ± 1% 216µs ± 7% +10.42% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 200µs ± 4% 214µs ± 9% +6.99% (p=0.002 n=9+8) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 223µs ± 2% 247µs ± 9% +10.60% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 612µs ±23% 855µs ±52% +39.60% (p=0.001 n=9+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 11.3ms ±53% 10.7ms ±50% ~ (p=0.739 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 12.0ms ±37% 10.4ms ±65% ~ (p=0.853 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 11.3ms ±40% 10.4ms ±49% ~ (p=0.631 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 3.80ms ±45% 3.69ms ±27% ~ (p=0.931 n=6+5) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 23.0µs ± 9% 22.6µs ± 4% ~ (p=0.059 n=8+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 25.9µs ± 5% 24.8µs ± 4% -4.17% (p=0.006 n=10+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 42.7µs ±13% 38.9µs ± 1% -8.85% (p=0.000 n=9+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 80.8µs ±12% 84.9µs ± 9% ~ (p=0.114 n=8+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 64.3µs ±16% 22.1µs ±23% -65.64% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 61.0µs ±34% 23.2µs ± 8% -62.04% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 62.2µs ±14% 25.7µs ±13% -58.66% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 359µs ±60% 325µs ±60% ~ (p=0.739 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 242µs ±21% 13µs ±28% -94.49% (p=0.000 n=10+8) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 229µs ±23% 18µs ±70% -92.02% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 238µs ±28% 20µs ±109% -91.38% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 265µs ±20% 77µs ±62% -71.04% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 25.5µs ± 3% 24.7µs ± 7% ~ (p=0.143 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 27.4µs ± 2% 27.9µs ± 6% ~ (p=0.280 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 50.2µs ± 7% 47.4µs ±10% ~ (p=0.156 n=9+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 98.2µs ± 9% 94.6µs ±10% ~ (p=0.218 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 82.9µs ±13% 32.1µs ±22% -61.30% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 92.2µs ±11% 33.7µs ±12% -63.42% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 88.3µs ±22% 39.4µs ±14% -55.36% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 106µs ±18% 78µs ±24% -26.20% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 360µs ±24% 29µs ±53% -91.91% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 353µs ±16% 50µs ±70% -85.72% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 381µs ±20% 47µs ±111% -87.64% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 434µs ±19% 113µs ±41% -74.04% (p=0.000 n=10+10) name old alloc/op new alloc/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 4.82kB ± 0% 4.26kB ± 1% -11.62% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 4.99kB ± 0% 4.41kB ± 1% -11.56% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 8.45kB ± 0% 7.87kB ± 0% -6.88% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 55.0kB ± 0% 54.5kB ± 0% -0.81% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 29.1kB ± 0% 21.7kB ± 2% -25.56% (p=0.000 n=9+9) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 29.3kB ± 1% 21.8kB ± 2% -25.74% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 31.3kB ± 1% 23.6kB ± 1% -24.50% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 92.5kB ± 5% 84.7kB ± 3% -8.50% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 324kB ±29% 222kB ±44% -31.33% (p=0.007 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 308kB ±32% 174kB ±14% -43.56% (p=0.000 n=10+8) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 298kB ±23% 178kB ±36% -40.26% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 362kB ± 6% 248kB ± 6% -31.54% (p=0.004 n=6+5) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 5.15kB ± 3% 4.64kB ± 2% -9.92% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 5.36kB ± 1% 4.75kB ± 5% -11.42% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 8.15kB ± 4% 7.53kB ± 1% -7.62% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 33.2kB ± 5% 33.2kB ± 7% ~ (p=0.829 n=8+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 20.1kB ± 7% 5.8kB ±13% -70.90% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 19.8kB ±14% 6.2kB ± 5% -68.87% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 21.7kB ± 6% 8.0kB ± 7% -63.20% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 98.5kB ±44% 81.8kB ±48% ~ (p=0.143 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 130kB ± 4% 4kB ± 9% -96.69% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 131kB ± 4% 5kB ±21% -96.48% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 132kB ± 4% 6kB ±10% -95.39% (p=0.000 n=10+8) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 151kB ± 4% 26kB ±10% -82.46% (p=0.000 n=9+9) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 5.92kB ± 3% 5.32kB ± 2% -10.01% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 6.09kB ± 2% 5.48kB ± 2% -10.00% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 9.61kB ± 1% 9.00kB ± 0% -6.29% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 33.4kB ± 7% 32.2kB ± 5% -3.60% (p=0.037 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 22.3kB ±10% 9.0kB ±16% -59.78% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 23.6kB ± 6% 8.5kB ±20% -63.76% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 24.2kB ± 9% 11.5kB ± 4% -52.34% (p=0.000 n=10+8) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 44.2kB ± 6% 30.8kB ± 9% -30.24% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 144kB ± 4% 10kB ±24% -93.39% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 146kB ± 1% 11kB ±37% -92.14% (p=0.000 n=7+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 149kB ± 3% 11kB ±12% -92.28% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 171kB ± 4% 34kB ±12% -80.00% (p=0.000 n=10+10) name old allocs/op new allocs/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 95.0 ± 0% 74.0 ± 0% -22.11% (p=0.001 n=8+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 100 ± 0% 78 ± 1% -21.70% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 153 ± 0% 131 ± 2% -14.25% (p=0.000 n=6+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 799 ± 2% 797 ± 4% ~ (p=0.956 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 438 ± 6% 167 ± 0% -61.86% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 446 ± 5% 172 ± 0% -61.38% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 506 ± 4% 232 ± 1% -54.21% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 1.31k ± 5% 0.97k ± 4% -26.20% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 5.06k ± 1% 1.09k ± 2% -78.53% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 5.02k ± 3% 1.08k ± 0% -78.45% (p=0.000 n=10+8) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 5.09k ± 3% 1.15k ± 2% -77.48% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 5.83k ± 1% 1.87k ± 3% -68.02% (p=0.004 n=6+5) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 103 ± 2% 82 ± 1% -20.83% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 107 ± 0% 86 ± 0% -19.63% (p=0.000 n=8+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 164 ± 1% 139 ± 0% -15.45% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 820 ± 1% 789 ± 1% -3.70% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 475 ± 0% 94 ± 3% -80.15% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 481 ± 0% 100 ± 2% -79.26% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 549 ± 0% 161 ± 2% -70.69% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 1.61k ±19% 1.19k ±25% -26.05% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 4.12k ± 0% 0.08k ± 2% -98.02% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 4.14k ± 0% 0.09k ± 3% -97.90% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 4.19k ± 0% 0.15k ± 3% -96.52% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 4.82k ± 1% 0.74k ± 1% -84.58% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 112 ± 4% 90 ± 3% -19.45% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 116 ± 2% 95 ± 2% -17.90% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 170 ± 3% 148 ± 3% -12.99% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 800 ± 2% 772 ± 2% -3.50% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 480 ± 3% 118 ± 3% -75.32% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 479 ± 2% 123 ± 3% -74.33% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 542 ± 1% 183 ± 3% -66.34% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 1.19k ± 1% 0.79k ± 1% -33.41% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 4.21k ± 1% 0.13k ±21% -96.83% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 4.23k ± 1% 0.15k ±17% -96.48% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 4.27k ± 0% 0.19k ± 6% -95.51% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 4.89k ± 1% 0.79k ± 2% -83.80% (p=0.000 n=10+10)
2022-01-17 17:41:51 +00:00
// Seek continues iteration until false is returned from f.
// Key and value slices should not be modified.
// Seek can guarantee that key-value items are sorted by key in ascending way.
Seek(rng SeekRange, f func(k, v []byte) bool)
// SeekGC is similar to Seek, but the function should return true if current
// KV pair should be kept and false if it's to be deleted; there is no way to
// do an early exit here. SeekGC only works with the current Store, it won't
// go down to layers below and it takes a full write lock, so use it carefully.
SeekGC(rng SeekRange, keep func(k, v []byte) bool) error
Close() error
}
// KeyPrefix is a constant byte added as a prefix for each key
// stored.
KeyPrefix uint8
)
func seekRangeToPrefixes(sr SeekRange) *util.Range {
var (
rang *util.Range
start = make([]byte, len(sr.Prefix)+len(sr.Start))
)
copy(start, sr.Prefix)
copy(start[len(sr.Prefix):], sr.Start)
if !sr.Backwards {
rang = util.BytesPrefix(sr.Prefix)
rang.Start = start
} else {
rang = util.BytesPrefix(start)
rang.Start = sr.Prefix
}
return rang
}
// NewStore creates storage with preselected in configuration database type.
func NewStore(cfg dbconfig.DBConfiguration) (Store, error) {
var store Store
var err error
switch cfg.Type {
2022-10-05 06:05:36 +00:00
case dbconfig.LevelDB:
store, err = NewLevelDBStore(cfg.LevelDBOptions)
2022-10-05 06:05:36 +00:00
case dbconfig.InMemoryDB:
store = NewMemoryStore()
2022-10-05 06:05:36 +00:00
case dbconfig.BoltDB:
store, err = NewBoltDBStore(cfg.BoltDBOptions)
default:
return nil, fmt.Errorf("unknown storage: %s", cfg.Type)
}
return store, err
}
// BatchToOperations converts a batch of changes into array of dboper.Operation.
func BatchToOperations(batch *MemBatch) []dboper.Operation {
size := len(batch.Put) + len(batch.Deleted)
ops := make([]dboper.Operation, 0, size)
for i := range batch.Put {
key := batch.Put[i].Key
if len(key) == 0 || key[0] != byte(STStorage) && key[0] != byte(STTempStorage) {
continue
}
op := "Added"
if batch.Put[i].Exists {
op = "Changed"
}
ops = append(ops, dboper.Operation{
State: op,
Key: key[1:],
Value: batch.Put[i].Value,
})
}
for i := range batch.Deleted {
key := batch.Deleted[i].Key
if len(key) == 0 || !batch.Deleted[i].Exists ||
key[0] != byte(STStorage) && key[0] != byte(STTempStorage) {
continue
}
ops = append(ops, dboper.Operation{
State: "Deleted",
Key: key[1:],
})
}
return ops
}