From d8cf87949991c0ee5ef504af4bb398cce524deaf Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 18 Feb 2022 11:41:27 +0300 Subject: [PATCH 01/11] dao: deduplicate DeleteBlock, no functional changes --- pkg/core/dao/dao.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index b6f6d3829..cd4221e1b 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -375,7 +375,10 @@ func (dao *Simple) makeStorageItemKey(id int32, key []byte) []byte { // GetBlock returns Block by the given hash if it exists in the store. func (dao *Simple) GetBlock(hash util.Uint256) (*block.Block, error) { - key := dao.makeExecutableKey(hash) + return dao.getBlock(dao.makeExecutableKey(hash)) +} + +func (dao *Simple) getBlock(key []byte) (*block.Block, error) { b, err := dao.Store.Get(key) if err != nil { return nil, err @@ -659,16 +662,8 @@ func (dao *Simple) StoreAsBlock(block *block.Block, aer1 *state.AppExecResult, a // using private MemCached instance here. func (dao *Simple) DeleteBlock(h util.Uint256) error { key := dao.makeExecutableKey(h) - bs, err := dao.Store.Get(key) - if err != nil { - return err - } - r := io.NewBinReaderFromBuf(bs) - if r.ReadB() != storage.ExecBlock { - return errors.New("internal DB inconsistency") - } - b, err := block.NewTrimmedFromReader(dao.Version.StateRootInHeader, r) + b, err := dao.getBlock(key) if err != nil { return err } From b60d4ff19177bf0c8137dab50ce3d43090431ccc Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 18 Feb 2022 11:55:06 +0300 Subject: [PATCH 02/11] dao: deduplicate header->KV conversion --- pkg/core/blockchain.go | 16 +++++----------- pkg/core/dao/dao.go | 27 ++++++++++++++++++++------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index b629a0373..9d127721f 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -932,7 +932,6 @@ func (bc *Blockchain) addHeaders(verify bool, headers ...*block.Header) error { } } - buf := io.NewBufBinWriter() bc.headerHashesLock.Lock() defer bc.headerHashesLock.Unlock() oldlen := len(bc.headerHashes) @@ -941,21 +940,16 @@ func (bc *Blockchain) addHeaders(verify bool, headers ...*block.Header) error { if int(h.Index) != len(bc.headerHashes) { continue } - bc.headerHashes = append(bc.headerHashes, h.Hash()) - buf.WriteB(storage.ExecBlock) - h.EncodeBinary(buf.BinWriter) - buf.BinWriter.WriteB(0) - if buf.Err != nil { - return buf.Err + err = batch.StoreHeader(h) + if err != nil { + return err } - - key := storage.AppendPrefix(storage.DataExecutable, h.Hash().BytesBE()) - batch.Store.Put(key, buf.Bytes()) - buf.Reset() + bc.headerHashes = append(bc.headerHashes, h.Hash()) lastHeader = h } if oldlen != len(bc.headerHashes) { + buf := io.NewBufBinWriter() for int(lastHeader.Index)-headerBatchCount >= int(bc.storedHeaderCount) { buf.WriteArray(bc.headerHashes[bc.storedHeaderCount : bc.storedHeaderCount+headerBatchCount]) if buf.Err != nil { diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index cd4221e1b..a69ac0355 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -668,14 +668,10 @@ func (dao *Simple) DeleteBlock(h util.Uint256) error { return err } - w := dao.getDataBuf() - w.WriteB(storage.ExecBlock) - b.Header.EncodeBinary(w.BinWriter) - w.BinWriter.WriteB(0) - if w.Err != nil { - return w.Err + err = dao.storeHeader(key, &b.Header) + if err != nil { + return err } - dao.Store.Put(key, w.Bytes()) for _, tx := range b.Transactions { copy(key[1:], tx.Hash().BytesBE()) @@ -692,6 +688,23 @@ func (dao *Simple) DeleteBlock(h util.Uint256) error { return nil } +// StoreHeader saves block header into the store. +func (dao *Simple) StoreHeader(h *block.Header) error { + return dao.storeHeader(dao.makeExecutableKey(h.Hash()), h) +} + +func (dao *Simple) storeHeader(key []byte, h *block.Header) error { + buf := dao.getDataBuf() + buf.WriteB(storage.ExecBlock) + h.EncodeBinary(buf.BinWriter) + buf.BinWriter.WriteB(0) + if buf.Err != nil { + return buf.Err + } + dao.Store.Put(key, buf.Bytes()) + return nil +} + // StoreAsCurrentBlock stores a hash of the given block with prefix // SYSCurrentBlock. It can reuse given buffer for the purpose of value // serialization. From 5402e654d13991acfa5f4e4e863028cde7e4202a Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 18 Feb 2022 13:18:00 +0300 Subject: [PATCH 03/11] core: don't create useless DAO layer in GetTestVM We're already wrapping in interop.NewContext. --- pkg/core/blockchain.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 9d127721f..00690e6be 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -2140,8 +2140,7 @@ func (bc *Blockchain) GetEnrollments() ([]state.Validator, error) { // GetTestVM returns an interop context with VM set up for a test run. func (bc *Blockchain) GetTestVM(t trigger.Type, tx *transaction.Transaction, b *block.Block) *interop.Context { - d := bc.dao.GetPrivate() - systemInterop := bc.newInteropContext(t, d, b, tx) + systemInterop := bc.newInteropContext(t, bc.dao, b, tx) vm := systemInterop.SpawnVM() vm.SetPriceGetter(systemInterop.GetPrice) vm.LoadToken = contract.LoadToken(systemInterop) From e864768c885974b549cbd0195533f0def54a2365 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 18 Feb 2022 14:18:56 +0300 Subject: [PATCH 04/11] dao: simplify NewPrivate --- pkg/core/dao/dao.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index a69ac0355..ec9c44dce 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -68,17 +68,13 @@ func (dao *Simple) GetWrapped() *Simple { // GetPrivate returns new DAO instance with another layer of private // MemCachedStore around the current DAO Store. func (dao *Simple) GetPrivate() *Simple { - st := storage.NewPrivateMemCachedStore(dao.Store) - d := newSimple(st, dao.Version.StateRootInHeader, dao.Version.P2PSigExtensions) - d.Version = dao.Version - if dao.keyBuf != nil { // This one is private. - d.keyBuf = dao.keyBuf // Thus we can reuse its buffer. - } else { + d := &Simple{} + *d = *dao // Inherit everything... + d.Store = storage.NewPrivateMemCachedStore(dao.Store) // except storage, wrap another layer. + if d.keyBuf == nil { d.keyBuf = make([]byte, 0, 1+4+storage.MaxStorageKeyLen) // Prefix, uint32, key. } - if dao.dataBuf != nil { // This one is private. - d.dataBuf = dao.dataBuf // Thus we can reuse its buffer. - } else { + if dao.dataBuf == nil { d.dataBuf = io.NewBufBinWriter() } return d From 1ca918e6310f73fd44454c8ed68f42979cd64a5f Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 18 Feb 2022 14:24:45 +0300 Subject: [PATCH 05/11] dao: delay buffer creation until it's needed Verification contexts don't ever touch the storage, so these allocations can be avoided for them. --- pkg/core/dao/dao.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index ec9c44dce..37e8a8d6d 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -31,6 +31,7 @@ var ( type Simple struct { Version Version Store *storage.MemCachedStore + private bool keyBuf []byte dataBuf *io.BufBinWriter } @@ -71,12 +72,7 @@ func (dao *Simple) GetPrivate() *Simple { d := &Simple{} *d = *dao // Inherit everything... d.Store = storage.NewPrivateMemCachedStore(dao.Store) // except storage, wrap another layer. - if d.keyBuf == nil { - d.keyBuf = make([]byte, 0, 1+4+storage.MaxStorageKeyLen) // Prefix, uint32, key. - } - if dao.dataBuf == nil { - d.dataBuf = io.NewBufBinWriter() - } + d.private = true return d } @@ -748,14 +744,20 @@ func (dao *Simple) StoreAsTransaction(tx *transaction.Transaction, index uint32, } func (dao *Simple) getKeyBuf(len int) []byte { - if dao.keyBuf != nil { // Private DAO. + if dao.private { + if dao.keyBuf == nil { + dao.keyBuf = make([]byte, 0, 1+4+storage.MaxStorageKeyLen) // Prefix, uint32, key. + } return dao.keyBuf[:len] // Should have enough capacity. } return make([]byte, len) } func (dao *Simple) getDataBuf() *io.BufBinWriter { - if dao.dataBuf != nil { + if dao.private { + if dao.dataBuf == nil { + dao.dataBuf = io.NewBufBinWriter() + } dao.dataBuf.Reset() return dao.dataBuf } From 600da6909c411ce4130e3ceee7c1778b633f44bd Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 18 Feb 2022 14:35:17 +0300 Subject: [PATCH 06/11] storage: put uint32 into keys using in big endianness Which allows to iterate over the contents easily. --- pkg/core/blockchain.go | 2 +- pkg/core/dao/dao.go | 23 ++++------------------- pkg/core/storage/store.go | 2 +- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 00690e6be..751d916bf 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -46,7 +46,7 @@ import ( // Tuning parameters. const ( headerBatchCount = 2000 - version = "0.2.3" + version = "0.2.4" defaultInitialGAS = 52000000_00000000 defaultGCPeriod = 10000 diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index 37e8a8d6d..23ca3bb68 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -6,7 +6,6 @@ import ( "encoding/binary" "errors" iocore "io" - "sort" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -506,33 +505,19 @@ func (dao *Simple) GetStateSyncCurrentBlockHeight() (uint32, error) { // GetHeaderHashes returns a sorted list of header hashes retrieved from // the given underlying store. func (dao *Simple) GetHeaderHashes() ([]util.Uint256, error) { - hashMap := make(map[uint32][]util.Uint256) + var hashes = make([]util.Uint256, 0) + dao.Store.Seek(storage.SeekRange{ Prefix: storage.IXHeaderHashList.Bytes(), }, func(k, v []byte) bool { - storedCount := binary.LittleEndian.Uint32(k[1:]) - hashes, err := read2000Uint256Hashes(v) + newHashes, err := read2000Uint256Hashes(v) if err != nil { panic(err) } - hashMap[storedCount] = hashes + hashes = append(hashes, newHashes...) return true }) - var ( - hashes = make([]util.Uint256, 0, len(hashMap)) - sortedKeys = make([]uint32, 0, len(hashMap)) - ) - - for k := range hashMap { - sortedKeys = append(sortedKeys, k) - } - sort.Slice(sortedKeys, func(i, j int) bool { return sortedKeys[i] < sortedKeys[j] }) - - for _, key := range sortedKeys { - hashes = append(hashes[:key], hashMap[key]...) - } - return hashes, nil } diff --git a/pkg/core/storage/store.go b/pkg/core/storage/store.go index 8d4bfad9d..b6cfe80bf 100644 --- a/pkg/core/storage/store.go +++ b/pkg/core/storage/store.go @@ -125,7 +125,7 @@ func AppendPrefix(k KeyPrefix, b []byte) []byte { // AppendPrefixInt(SYSCurrentHeader, 10001) func AppendPrefixInt(k KeyPrefix, n int) []byte { b := make([]byte, 4) - binary.LittleEndian.PutUint32(b, uint32(n)) + binary.BigEndian.PutUint32(b, uint32(n)) return AppendPrefix(k, b) } From de2579ec0793645229afbc36765ed4d94f2a26e0 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 18 Feb 2022 14:38:51 +0300 Subject: [PATCH 07/11] dao: put contract IDs into keys using big endianness We don't have a need to iterate over them at the moment, but since we're changing the DB format in the next release anyway let's add this ability also, just in case. --- pkg/core/blockchain.go | 2 +- pkg/core/dao/dao.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 751d916bf..28e7715f4 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -46,7 +46,7 @@ import ( // Tuning parameters. const ( headerBatchCount = 2000 - version = "0.2.4" + version = "0.2.5" defaultInitialGAS = 52000000_00000000 defaultGCPeriod = 10000 diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index 23ca3bb68..505e589dc 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -99,7 +99,7 @@ func (dao *Simple) putWithBuffer(entity io.Serializable, key []byte, buf *io.Buf func (dao *Simple) makeContractIDKey(id int32) []byte { key := dao.getKeyBuf(5) key[0] = byte(storage.STContractID) - binary.LittleEndian.PutUint32(key[1:], uint32(id)) + binary.BigEndian.PutUint32(key[1:], uint32(id)) return key } From d2db58d7481acb8b4a317487c8719850279a5aa6 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 18 Feb 2022 14:54:05 +0300 Subject: [PATCH 08/11] dao: move header hash store logic out of the core Which allows for more efficient buffer use along the way. --- pkg/core/blockchain.go | 11 ++++------- pkg/core/dao/dao.go | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 28e7715f4..346905870 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -949,15 +949,12 @@ func (bc *Blockchain) addHeaders(verify bool, headers ...*block.Header) error { } if oldlen != len(bc.headerHashes) { - buf := io.NewBufBinWriter() for int(lastHeader.Index)-headerBatchCount >= int(bc.storedHeaderCount) { - buf.WriteArray(bc.headerHashes[bc.storedHeaderCount : bc.storedHeaderCount+headerBatchCount]) - if buf.Err != nil { - return buf.Err + err = batch.StoreHeaderHashes(bc.headerHashes[bc.storedHeaderCount:bc.storedHeaderCount+headerBatchCount], + bc.storedHeaderCount) + if err != nil { + return err } - - key := storage.AppendPrefixInt(storage.IXHeaderHashList, int(bc.storedHeaderCount)) - batch.Store.Put(key, buf.Bytes()) bc.storedHeaderCount += headerBatchCount } diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index 505e589dc..af781d713 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -590,6 +590,25 @@ func read2000Uint256Hashes(b []byte) ([]util.Uint256, error) { return hashes, nil } +func (dao *Simple) mkHeaderHashKey(h uint32) []byte { + b := dao.getKeyBuf(1 + 4) + b[0] = byte(storage.IXHeaderHashList) + binary.BigEndian.PutUint32(b[1:], h) + return b +} + +// StoreHeaderHashes pushes a batch of header hashes into the store. +func (dao *Simple) StoreHeaderHashes(hashes []util.Uint256, height uint32) error { + key := dao.mkHeaderHashKey(height) + buf := dao.getDataBuf() + buf.WriteArray(hashes) + if buf.Err != nil { + return buf.Err + } + dao.Store.Put(key, buf.Bytes()) + return nil +} + // HasTransaction returns nil if the given store does not contain the given // Transaction hash. It returns an error in case if transaction is in chain // or in the list of conflicting transactions. From 522229d731ff375e79681f7ec6212d7b3a20f007 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 18 Feb 2022 14:59:59 +0300 Subject: [PATCH 09/11] storage: drop AppendPrefix/AppendPrefixInt APIs We're not using them anymore and they allocate. --- pkg/core/blockchain_test.go | 4 +++- pkg/core/storage/store.go | 18 ---------------- pkg/core/storage/store_test.go | 39 ---------------------------------- 3 files changed, 3 insertions(+), 58 deletions(-) diff --git a/pkg/core/blockchain_test.go b/pkg/core/blockchain_test.go index 14c687a19..079b4efd3 100644 --- a/pkg/core/blockchain_test.go +++ b/pkg/core/blockchain_test.go @@ -124,8 +124,10 @@ func TestAddBlock(t *testing.T) { _, err = bc.persist(false) require.NoError(t, err) + key := make([]byte, 1+util.Uint256Size) + key[0] = byte(storage.DataExecutable) for _, block := range blocks { - key := storage.AppendPrefix(storage.DataExecutable, block.Hash().BytesBE()) + copy(key[1:], block.Hash().BytesBE()) _, err := bc.dao.Store.Get(key) require.NoErrorf(t, err, "block %s not persisted", block.Hash()) } diff --git a/pkg/core/storage/store.go b/pkg/core/storage/store.go index b6cfe80bf..0812a5f08 100644 --- a/pkg/core/storage/store.go +++ b/pkg/core/storage/store.go @@ -1,7 +1,6 @@ package storage import ( - "encoding/binary" "errors" "fmt" @@ -112,23 +111,6 @@ func (k KeyPrefix) Bytes() []byte { return []byte{byte(k)} } -// AppendPrefix appends byteslice b to the given KeyPrefix. -// AppendKeyPrefix(SYSVersion, []byte{0x00, 0x01}). -func AppendPrefix(k KeyPrefix, b []byte) []byte { - dest := make([]byte, len(b)+1) - dest[0] = byte(k) - copy(dest[1:], b) - return dest -} - -// AppendPrefixInt append int n to the given KeyPrefix. -// AppendPrefixInt(SYSCurrentHeader, 10001) -func AppendPrefixInt(k KeyPrefix, n int) []byte { - b := make([]byte, 4) - binary.BigEndian.PutUint32(b, uint32(n)) - return AppendPrefix(k, b) -} - func seekRangeToPrefixes(sr SeekRange) *util.Range { var ( rang *util.Range diff --git a/pkg/core/storage/store_test.go b/pkg/core/storage/store_test.go index a07b97ece..7191e46ad 100644 --- a/pkg/core/storage/store_test.go +++ b/pkg/core/storage/store_test.go @@ -3,48 +3,9 @@ package storage import ( "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -var ( - prefixes = []KeyPrefix{ - DataExecutable, - DataMPT, - STStorage, - IXHeaderHashList, - SYSCurrentBlock, - SYSCurrentHeader, - SYSVersion, - } - - expected = []uint8{ - 0x01, - 0x03, - 0x70, - 0x80, - 0xc0, - 0xc1, - 0xf0, - } -) - -func TestAppendPrefix(t *testing.T) { - for i := 0; i < len(expected); i++ { - value := []byte{0x01, 0x02} - prefix := AppendPrefix(prefixes[i], value) - assert.Equal(t, KeyPrefix(expected[i]), KeyPrefix(prefix[0])) - } -} - -func TestAppendPrefixInt(t *testing.T) { - for i := 0; i < len(expected); i++ { - value := 2000 - prefix := AppendPrefixInt(prefixes[i], value) - assert.Equal(t, KeyPrefix(expected[i]), KeyPrefix(prefix[0])) - } -} - func TestBatchToOperations(t *testing.T) { b := &MemBatch{ Put: []KeyValueExists{ From 7223caf36953efd597303f3c7cd7a2aa91831c6c Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 18 Feb 2022 15:04:57 +0300 Subject: [PATCH 10/11] dao: improve PutCurrentHeader logic Move serialization out of the core. --- pkg/core/blockchain.go | 11 ++--------- pkg/core/dao/dao.go | 7 +++++-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 346905870..624b9932c 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -325,7 +325,7 @@ func (bc *Blockchain) init() error { return err } bc.headerHashes = []util.Uint256{genesisBlock.Hash()} - bc.dao.PutCurrentHeader(hashAndIndexToBytes(genesisBlock.Hash(), genesisBlock.Index)) + bc.dao.PutCurrentHeader(genesisBlock.Hash(), genesisBlock.Index) if err := bc.stateRoot.Init(0); err != nil { return fmt.Errorf("can't init MPT: %w", err) } @@ -958,7 +958,7 @@ func (bc *Blockchain) addHeaders(verify bool, headers ...*block.Header) error { bc.storedHeaderCount += headerBatchCount } - batch.Store.Put(storage.SYSCurrentHeader.Bytes(), hashAndIndexToBytes(lastHeader.Hash(), lastHeader.Index)) + batch.PutCurrentHeader(lastHeader.Hash(), lastHeader.Index) updateHeaderHeightMetric(len(bc.headerHashes) - 1) if _, err = batch.Persist(); err != nil { return err @@ -2299,13 +2299,6 @@ func (bc *Blockchain) ManagementContractHash() util.Uint160 { return bc.contracts.Management.Hash } -func hashAndIndexToBytes(h util.Uint256, index uint32) []byte { - buf := io.NewBufBinWriter() - buf.WriteBytes(h.BytesLE()) - buf.WriteU32LE(index) - return buf.Bytes() -} - func (bc *Blockchain) newInteropContext(trigger trigger.Type, d *dao.Simple, block *block.Block, tx *transaction.Transaction) *interop.Context { ic := interop.NewContext(trigger, bc, d, bc.contracts.Management.GetContract, bc.contracts.Contracts, block, tx, bc.log) ic.Functions = systemInterops diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index af781d713..4b7520d45 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -559,8 +559,11 @@ func (dao *Simple) PutVersion(v Version) { } // PutCurrentHeader stores current header. -func (dao *Simple) PutCurrentHeader(hashAndIndex []byte) { - dao.Store.Put(storage.SYSCurrentHeader.Bytes(), hashAndIndex) +func (dao *Simple) PutCurrentHeader(h util.Uint256, index uint32) { + buf := dao.getDataBuf() + buf.WriteBytes(h.BytesLE()) + buf.WriteU32LE(index) + dao.Store.Put(storage.SYSCurrentHeader.Bytes(), buf.Bytes()) } // PutStateSyncPoint stores current state synchronisation point P. From 7d6f08733710f4919d3440e77e14285e937361ed Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 18 Feb 2022 15:19:57 +0300 Subject: [PATCH 11/11] storage: drop (KeyPrefix).Bytes() API It allocates and most of the time we can avoid that. --- pkg/core/blockchain.go | 4 ++-- pkg/core/blockchain_test.go | 19 +++++++++------- pkg/core/dao/dao.go | 36 ++++++++++++++++++------------- pkg/core/dao/dao_test.go | 4 ++-- pkg/core/statesync/module_test.go | 4 ++-- pkg/core/statesync_test.go | 4 ++-- pkg/core/storage/store.go | 5 ----- 7 files changed, 40 insertions(+), 36 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 624b9932c..1d51823f8 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -406,7 +406,7 @@ func (bc *Blockchain) init() error { } // Check whether StateJump stage is in the storage and continue interrupted state jump if so. - jumpStage, err := bc.dao.Store.Get(storage.SYSStateJumpStage.Bytes()) + jumpStage, err := bc.dao.Store.Get([]byte{byte(storage.SYSStateJumpStage)}) if err == nil { if !(bc.GetConfig().P2PStateExchangeExtensions && bc.GetConfig().RemoveUntraceableBlocks) { return errors.New("state jump was not completed, but P2PStateExchangeExtensions are disabled or archival node capability is on. " + @@ -500,7 +500,7 @@ func (bc *Blockchain) jumpToStateInternal(p uint32, stage stateJumpStage) error bc.log.Info("jumping to state sync point", zap.Uint32("state sync point", p)) - jumpStageKey := storage.SYSStateJumpStage.Bytes() + jumpStageKey := []byte{byte(storage.SYSStateJumpStage)} switch stage { case none: bc.dao.Store.Put(jumpStageKey, []byte{byte(stateJumpStarted)}) diff --git a/pkg/core/blockchain_test.go b/pkg/core/blockchain_test.go index 079b4efd3..39710482f 100644 --- a/pkg/core/blockchain_test.go +++ b/pkg/core/blockchain_test.go @@ -1853,7 +1853,9 @@ func TestBlockchain_InitWithIncompleteStateJump(t *testing.T) { if bcSpout.dao.Version.StoragePrefix == tempPrefix { tempPrefix = storage.STStorage } - bcSpout.dao.Store.Seek(storage.SeekRange{Prefix: bcSpout.dao.Version.StoragePrefix.Bytes()}, func(k, v []byte) bool { + bPrefix := make([]byte, 1) + bPrefix[0] = byte(bcSpout.dao.Version.StoragePrefix) + bcSpout.dao.Store.Seek(storage.SeekRange{Prefix: bPrefix}, func(k, v []byte) bool { key := slice.Copy(k) key[0] = byte(tempPrefix) value := slice.Copy(v) @@ -1880,34 +1882,35 @@ func TestBlockchain_InitWithIncompleteStateJump(t *testing.T) { c.ProtocolConfiguration.KeepOnlyLatestState = true } // manually store statejump stage to check statejump recover process + bPrefix[0] = byte(storage.SYSStateJumpStage) t.Run("invalid RemoveUntraceableBlocks setting", func(t *testing.T) { - bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{byte(stateJumpStarted)}) + bcSpout.dao.Store.Put(bPrefix, []byte{byte(stateJumpStarted)}) checkNewBlockchainErr(t, func(c *config.Config) { boltCfg(c) c.ProtocolConfiguration.RemoveUntraceableBlocks = false }, bcSpout.dao.Store, true) }) t.Run("invalid state jump stage format", func(t *testing.T) { - bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{0x01, 0x02}) + bcSpout.dao.Store.Put(bPrefix, []byte{0x01, 0x02}) checkNewBlockchainErr(t, boltCfg, bcSpout.dao.Store, true) }) t.Run("missing state sync point", func(t *testing.T) { - bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{byte(stateJumpStarted)}) + bcSpout.dao.Store.Put(bPrefix, []byte{byte(stateJumpStarted)}) checkNewBlockchainErr(t, boltCfg, bcSpout.dao.Store, true) }) t.Run("invalid state sync point", func(t *testing.T) { - bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{byte(stateJumpStarted)}) + bcSpout.dao.Store.Put(bPrefix, []byte{byte(stateJumpStarted)}) point := make([]byte, 4) binary.LittleEndian.PutUint32(point, uint32(len(bcSpout.headerHashes))) - bcSpout.dao.Store.Put(storage.SYSStateSyncPoint.Bytes(), point) + bcSpout.dao.Store.Put([]byte{byte(storage.SYSStateSyncPoint)}, point) checkNewBlockchainErr(t, boltCfg, bcSpout.dao.Store, true) }) for _, stage := range []stateJumpStage{stateJumpStarted, newStorageItemsAdded, genesisStateRemoved, 0x03} { t.Run(fmt.Sprintf("state jump stage %d", stage), func(t *testing.T) { - bcSpout.dao.Store.Put(storage.SYSStateJumpStage.Bytes(), []byte{byte(stage)}) + bcSpout.dao.Store.Put(bPrefix, []byte{byte(stage)}) point := make([]byte, 4) binary.LittleEndian.PutUint32(point, uint32(stateSyncPoint)) - bcSpout.dao.Store.Put(storage.SYSStateSyncPoint.Bytes(), point) + bcSpout.dao.Store.Put([]byte{byte(storage.SYSStateSyncPoint)}, point) shouldFail := stage == 0x03 // unknown stage checkNewBlockchainErr(t, spountCfg, bcSpout.dao.Store, shouldFail) }) diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index 4b7520d45..cae2622d6 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -448,12 +448,18 @@ func (v *Version) Bytes() []byte { return append([]byte(v.Value), '\x00', byte(v.StoragePrefix), mask) } +func (dao *Simple) mkKeyPrefix(k storage.KeyPrefix) []byte { + b := dao.getKeyBuf(1) + b[0] = byte(k) + return b +} + // GetVersion attempts to get the current version stored in the // underlying store. func (dao *Simple) GetVersion() (Version, error) { var version Version - data, err := dao.Store.Get(storage.SYSVersion.Bytes()) + data, err := dao.Store.Get(dao.mkKeyPrefix(storage.SYSVersion)) if err == nil { err = version.FromBytes(data) } @@ -463,7 +469,7 @@ func (dao *Simple) GetVersion() (Version, error) { // GetCurrentBlockHeight returns the current block height found in the // underlying store. func (dao *Simple) GetCurrentBlockHeight() (uint32, error) { - b, err := dao.Store.Get(storage.SYSCurrentBlock.Bytes()) + b, err := dao.Store.Get(dao.mkKeyPrefix(storage.SYSCurrentBlock)) if err != nil { return 0, err } @@ -474,7 +480,7 @@ func (dao *Simple) GetCurrentBlockHeight() (uint32, error) { // the underlying store. func (dao *Simple) GetCurrentHeaderHeight() (i uint32, h util.Uint256, err error) { var b []byte - b, err = dao.Store.Get(storage.SYSCurrentHeader.Bytes()) + b, err = dao.Store.Get(dao.mkKeyPrefix(storage.SYSCurrentHeader)) if err != nil { return } @@ -485,7 +491,7 @@ func (dao *Simple) GetCurrentHeaderHeight() (i uint32, h util.Uint256, err error // GetStateSyncPoint returns current state synchronisation point P. func (dao *Simple) GetStateSyncPoint() (uint32, error) { - b, err := dao.Store.Get(storage.SYSStateSyncPoint.Bytes()) + b, err := dao.Store.Get(dao.mkKeyPrefix(storage.SYSStateSyncPoint)) if err != nil { return 0, err } @@ -495,7 +501,7 @@ func (dao *Simple) GetStateSyncPoint() (uint32, error) { // GetStateSyncCurrentBlockHeight returns current block height stored during state // synchronisation process. func (dao *Simple) GetStateSyncCurrentBlockHeight() (uint32, error) { - b, err := dao.Store.Get(storage.SYSStateSyncCurrentBlockHeight.Bytes()) + b, err := dao.Store.Get(dao.mkKeyPrefix(storage.SYSStateSyncCurrentBlockHeight)) if err != nil { return 0, err } @@ -508,7 +514,7 @@ func (dao *Simple) GetHeaderHashes() ([]util.Uint256, error) { var hashes = make([]util.Uint256, 0) dao.Store.Seek(storage.SeekRange{ - Prefix: storage.IXHeaderHashList.Bytes(), + Prefix: dao.mkKeyPrefix(storage.IXHeaderHashList), }, func(k, v []byte) bool { newHashes, err := read2000Uint256Hashes(v) if err != nil { @@ -555,7 +561,7 @@ func (dao *Simple) GetTransaction(hash util.Uint256) (*transaction.Transaction, // PutVersion stores the given version in the underlying store. func (dao *Simple) PutVersion(v Version) { dao.Version = v - dao.Store.Put(storage.SYSVersion.Bytes(), v.Bytes()) + dao.Store.Put(dao.mkKeyPrefix(storage.SYSVersion), v.Bytes()) } // PutCurrentHeader stores current header. @@ -563,21 +569,21 @@ func (dao *Simple) PutCurrentHeader(h util.Uint256, index uint32) { buf := dao.getDataBuf() buf.WriteBytes(h.BytesLE()) buf.WriteU32LE(index) - dao.Store.Put(storage.SYSCurrentHeader.Bytes(), buf.Bytes()) + dao.Store.Put(dao.mkKeyPrefix(storage.SYSCurrentHeader), buf.Bytes()) } // PutStateSyncPoint stores current state synchronisation point P. func (dao *Simple) PutStateSyncPoint(p uint32) { - buf := dao.getKeyBuf(4) // It's very small, no point in using BufBinWriter. - binary.LittleEndian.PutUint32(buf, p) - dao.Store.Put(storage.SYSStateSyncPoint.Bytes(), buf) + buf := dao.getDataBuf() + buf.WriteU32LE(p) + dao.Store.Put(dao.mkKeyPrefix(storage.SYSStateSyncPoint), buf.Bytes()) } // PutStateSyncCurrentBlockHeight stores current block height during state synchronisation process. func (dao *Simple) PutStateSyncCurrentBlockHeight(h uint32) { - buf := dao.getKeyBuf(4) // It's very small, no point in using BufBinWriter. - binary.LittleEndian.PutUint32(buf, h) - dao.Store.Put(storage.SYSStateSyncCurrentBlockHeight.Bytes(), buf) + buf := dao.getDataBuf() + buf.WriteU32LE(h) + dao.Store.Put(dao.mkKeyPrefix(storage.SYSStateSyncCurrentBlockHeight), buf.Bytes()) } // read2000Uint256Hashes attempts to read 2000 Uint256 hashes from @@ -712,7 +718,7 @@ func (dao *Simple) StoreAsCurrentBlock(block *block.Block) { h := block.Hash() h.EncodeBinary(buf.BinWriter) buf.WriteU32LE(block.Index) - dao.Store.Put(storage.SYSCurrentBlock.Bytes(), buf.Bytes()) + dao.Store.Put(dao.mkKeyPrefix(storage.SYSCurrentBlock), buf.Bytes()) } // StoreAsTransaction stores given TX as DataTransaction. It also stores transactions diff --git a/pkg/core/dao/dao_test.go b/pkg/core/dao/dao_test.go index feb2e2055..6583319e7 100644 --- a/pkg/core/dao/dao_test.go +++ b/pkg/core/dao/dao_test.go @@ -132,14 +132,14 @@ func TestGetVersion(t *testing.T) { t.Run("invalid", func(t *testing.T) { dao := NewSimple(storage.NewMemoryStore(), false, false) - dao.Store.Put(storage.SYSVersion.Bytes(), []byte("0.1.2\x00x")) + dao.Store.Put([]byte{byte(storage.SYSVersion)}, []byte("0.1.2\x00x")) _, err := dao.GetVersion() require.Error(t, err) }) t.Run("old format", func(t *testing.T) { dao := NewSimple(storage.NewMemoryStore(), false, false) - dao.Store.Put(storage.SYSVersion.Bytes(), []byte("0.1.2")) + dao.Store.Put([]byte{byte(storage.SYSVersion)}, []byte("0.1.2")) version, err := dao.GetVersion() require.NoError(t, err) diff --git a/pkg/core/statesync/module_test.go b/pkg/core/statesync/module_test.go index 79e24a8ba..6894f0593 100644 --- a/pkg/core/statesync/module_test.go +++ b/pkg/core/statesync/module_test.go @@ -32,7 +32,7 @@ func TestModule_PR2019_discussion_r689629704(t *testing.T) { nodes = make(map[util.Uint256][]byte) expectedItems []storage.KeyValue ) - expectedStorage.Seek(storage.SeekRange{Prefix: storage.DataMPT.Bytes()}, func(k, v []byte) bool { + expectedStorage.Seek(storage.SeekRange{Prefix: []byte{byte(storage.DataMPT)}}, func(k, v []byte) bool { key := slice.Copy(k) value := slice.Copy(v) expectedItems = append(expectedItems, storage.KeyValue{ @@ -96,7 +96,7 @@ func TestModule_PR2019_discussion_r689629704(t *testing.T) { // Compare resulting storage items and refcounts. var actualItems []storage.KeyValue - expectedStorage.Seek(storage.SeekRange{Prefix: storage.DataMPT.Bytes()}, func(k, v []byte) bool { + expectedStorage.Seek(storage.SeekRange{Prefix: []byte{byte(storage.DataMPT)}}, func(k, v []byte) bool { key := slice.Copy(k) value := slice.Copy(v) actualItems = append(actualItems, storage.KeyValue{ diff --git a/pkg/core/statesync_test.go b/pkg/core/statesync_test.go index 9da8e5f2b..2be3b01a9 100644 --- a/pkg/core/statesync_test.go +++ b/pkg/core/statesync_test.go @@ -423,7 +423,7 @@ func TestStateSyncModule_RestoreBasicChain(t *testing.T) { // compare storage states fetchStorage := func(bc *Blockchain) []storage.KeyValue { var kv []storage.KeyValue - bc.dao.Store.Seek(storage.SeekRange{Prefix: bc.dao.Version.StoragePrefix.Bytes()}, func(k, v []byte) bool { + bc.dao.Store.Seek(storage.SeekRange{Prefix: []byte{byte(bc.dao.Version.StoragePrefix)}}, func(k, v []byte) bool { key := slice.Copy(k) value := slice.Copy(v) if key[0] == byte(storage.STTempStorage) { @@ -444,7 +444,7 @@ func TestStateSyncModule_RestoreBasicChain(t *testing.T) { // no temp items should be left require.Eventually(t, func() bool { var haveItems bool - bcBolt.dao.Store.Seek(storage.SeekRange{Prefix: storage.STStorage.Bytes()}, func(_, _ []byte) bool { + bcBolt.dao.Store.Seek(storage.SeekRange{Prefix: []byte{byte(storage.STStorage)}}, func(_, _ []byte) bool { haveItems = true return false }) diff --git a/pkg/core/storage/store.go b/pkg/core/storage/store.go index 0812a5f08..4d2b1a550 100644 --- a/pkg/core/storage/store.go +++ b/pkg/core/storage/store.go @@ -106,11 +106,6 @@ type ( KeyPrefix uint8 ) -// Bytes returns the bytes representation of KeyPrefix. -func (k KeyPrefix) Bytes() []byte { - return []byte{byte(k)} -} - func seekRangeToPrefixes(sr SeekRange) *util.Range { var ( rang *util.Range