From 987951441265b0885d29833035eeb7469156a431 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 29 Dec 2021 13:36:10 +0300 Subject: [PATCH 1/6] core: add benchmark for (*NEO).getGASPerVote --- pkg/core/helper_test.go | 22 +++++++ pkg/core/native_neo_test.go | 120 +++++++++++++++++++++++++++++++++++- 2 files changed, 141 insertions(+), 1 deletion(-) diff --git a/pkg/core/helper_test.go b/pkg/core/helper_test.go index f117c9825..1d52f6593 100644 --- a/pkg/core/helper_test.go +++ b/pkg/core/helper_test.go @@ -10,6 +10,7 @@ import ( "math/big" "os" "path" + "path/filepath" "strings" "testing" "time" @@ -68,6 +69,27 @@ func newTestChainWithCustomCfgAndStore(t testing.TB, st storage.Store, f func(*c return chain } +func newLevelDBForTesting(t testing.TB) storage.Store { + ldbDir := t.TempDir() + dbConfig := storage.DBConfiguration{ + Type: "leveldb", + LevelDBOptions: storage.LevelDBOptions{ + DataDirectoryPath: ldbDir, + }, + } + newLevelStore, err := storage.NewLevelDBStore(dbConfig.LevelDBOptions) + require.Nil(t, err, "NewLevelDBStore error") + return newLevelStore +} + +func newBoltStoreForTesting(t testing.TB) storage.Store { + d := t.TempDir() + testFileName := filepath.Join(d, "test_bolt_db") + boltDBStore, err := storage.NewBoltDBStore(storage.BoltDBOptions{FilePath: testFileName}) + require.NoError(t, err) + return boltDBStore +} + func initTestChain(t testing.TB, st storage.Store, f func(*config.Config)) *Blockchain { unitTestNetCfg, err := config.Load("../../config", testchain.Network()) require.NoError(t, err) diff --git a/pkg/core/native_neo_test.go b/pkg/core/native_neo_test.go index a31cf80a5..e96915250 100644 --- a/pkg/core/native_neo_test.go +++ b/pkg/core/native_neo_test.go @@ -1,6 +1,7 @@ package core import ( + "fmt" "math" "math/big" "sort" @@ -10,6 +11,7 @@ import ( "github.com/nspcc-dev/neo-go/internal/testchain" "github.com/nspcc-dev/neo-go/pkg/config/netmode" "github.com/nspcc-dev/neo-go/pkg/core/native" + "github.com/nspcc-dev/neo-go/pkg/core/storage" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/io" @@ -31,7 +33,7 @@ func setSigner(tx *transaction.Transaction, h util.Uint160) { }} } -func checkTxHalt(t *testing.T, bc *Blockchain, h util.Uint256) { +func checkTxHalt(t testing.TB, bc *Blockchain, h util.Uint256) { aer, err := bc.GetAppExecResults(h, trigger.Application) require.NoError(t, err) require.Equal(t, 1, len(aer)) @@ -476,3 +478,119 @@ func newAccountWithGAS(t *testing.T, bc *Blockchain) *wallet.Account { transferTokenFromMultisigAccount(t, bc, acc.PrivateKey().GetScriptHash(), bc.contracts.GAS.Hash, 1000_00000000) return acc } + +func BenchmarkNEO_GetGASPerVote(t *testing.B) { + var stores = map[string]func(testing.TB) storage.Store{ + "MemPS": func(t testing.TB) storage.Store { + return storage.NewMemoryStore() + }, + "BoltPS": newBoltStoreForTesting, + "LevelPS": newLevelDBForTesting, + } + for psName, newPS := range stores { + for nRewardRecords := 10; nRewardRecords <= 1000; nRewardRecords *= 10 { + for rewardDistance := 1; rewardDistance <= 1000; rewardDistance *= 10 { + t.Run(fmt.Sprintf("%s_%dRewardRecords_%dRewardDistance", psName, nRewardRecords, rewardDistance), func(t *testing.B) { + ps := newPS(t) + t.Cleanup(func() { ps.Close() }) + benchmarkGasPerVote(t, ps, nRewardRecords, rewardDistance) + }) + } + } + } +} + +func benchmarkGasPerVote(t *testing.B, ps storage.Store, nRewardRecords int, rewardDistance int) { + bc := newTestChainWithCustomCfgAndStore(t, ps, nil) + + neo := bc.contracts.NEO + tx := transaction.New([]byte{byte(opcode.PUSH1)}, 0) + ic := bc.newInteropContext(trigger.Application, bc.dao, nil, tx) + ic.SpawnVM() + ic.Block = bc.newBlock(tx) + + advanceChain := func(t *testing.B, count int) { + for i := 0; i < count; i++ { + require.NoError(t, bc.AddBlock(bc.newBlock())) + ic.Block.Index++ + } + } + + // Vote for new committee. + sz := testchain.CommitteeSize() + accs := make([]*wallet.Account, sz) + candidates := make(keys.PublicKeys, sz) + txs := make([]*transaction.Transaction, 0, len(accs)) + for i := 0; i < sz; i++ { + priv, err := keys.NewPrivateKey() + require.NoError(t, err) + candidates[i] = priv.PublicKey() + accs[i], err = wallet.NewAccount() + require.NoError(t, err) + require.NoError(t, neo.RegisterCandidateInternal(ic, candidates[i])) + + to := accs[i].Contract.ScriptHash() + w := io.NewBufBinWriter() + emit.AppCall(w.BinWriter, bc.contracts.NEO.Hash, "transfer", callflag.All, + neoOwner.BytesBE(), to.BytesBE(), + big.NewInt(int64(sz-i)*1000000).Int64(), nil) + emit.Opcodes(w.BinWriter, opcode.ASSERT) + emit.AppCall(w.BinWriter, bc.contracts.GAS.Hash, "transfer", callflag.All, + neoOwner.BytesBE(), to.BytesBE(), + int64(1_000_000_000), nil) + emit.Opcodes(w.BinWriter, opcode.ASSERT) + require.NoError(t, w.Err) + tx := transaction.New(w.Bytes(), 1000_000_000) + tx.ValidUntilBlock = bc.BlockHeight() + 1 + setSigner(tx, testchain.MultisigScriptHash()) + require.NoError(t, testchain.SignTx(bc, tx)) + txs = append(txs, tx) + } + require.NoError(t, bc.AddBlock(bc.newBlock(txs...))) + for _, tx := range txs { + checkTxHalt(t, bc, tx.Hash()) + } + for i := 0; i < sz; i++ { + priv := accs[i].PrivateKey() + h := priv.GetScriptHash() + setSigner(tx, h) + ic.VM.Load(priv.PublicKey().GetVerificationScript()) + require.NoError(t, neo.VoteInternal(ic, h, candidates[i])) + } + _, err := ic.DAO.Persist() + require.NoError(t, err) + + // Collect set of nRewardRecords reward records for each voter. + advanceChain(t, nRewardRecords*testchain.CommitteeSize()) + + // Transfer some more NEO to first voter to update his balance height. + to := accs[0].Contract.ScriptHash() + w := io.NewBufBinWriter() + emit.AppCall(w.BinWriter, bc.contracts.NEO.Hash, "transfer", callflag.All, + neoOwner.BytesBE(), to.BytesBE(), int64(1), nil) + emit.Opcodes(w.BinWriter, opcode.ASSERT) + require.NoError(t, w.Err) + tx = transaction.New(w.Bytes(), 1000_000_000) + tx.ValidUntilBlock = bc.BlockHeight() + 1 + setSigner(tx, testchain.MultisigScriptHash()) + require.NoError(t, testchain.SignTx(bc, tx)) + require.NoError(t, bc.AddBlock(bc.newBlock(tx))) + + aer, err := bc.GetAppExecResults(tx.Hash(), trigger.Application) + require.NoError(t, err) + require.Equal(t, 1, len(aer)) + require.Equal(t, vm.HaltState, aer[0].VMState, aer[0].FaultException) + + // Advance chain one more time to avoid same start/end rewarding bounds. + advanceChain(t, rewardDistance) + end := bc.BlockHeight() + + t.ResetTimer() + t.ReportAllocs() + t.StartTimer() + for i := 0; i < t.N; i++ { + _, err := neo.CalculateBonus(ic.DAO, to, end) + require.NoError(t, err) + } + t.StopTimer() +} From 5770a581c3fa66723c805c001214f59ff07098c7 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 28 Dec 2021 17:07:52 +0300 Subject: [PATCH 2/6] store: improve Seek tests After #2193 Seek results are sorted in an ascending way, so technically the test was needed to be fixed along with these changes. --- pkg/core/storage/memcached_store_test.go | 76 ++++++++++++------------ pkg/core/storage/storeandbatch_test.go | 60 +++++++------------ 2 files changed, 61 insertions(+), 75 deletions(-) diff --git a/pkg/core/storage/memcached_store_test.go b/pkg/core/storage/memcached_store_test.go index eb572d48f..f5c380238 100644 --- a/pkg/core/storage/memcached_store_test.go +++ b/pkg/core/storage/memcached_store_test.go @@ -138,33 +138,33 @@ func TestCachedSeek(t *testing.T) { // Given this prefix... goodPrefix = []byte{'f'} // these pairs should be found... - lowerKVs = []kvSeen{ - {[]byte("foo"), []byte("bar"), false}, - {[]byte("faa"), []byte("bra"), false}, + lowerKVs = []KeyValue{ + {[]byte("foo"), []byte("bar")}, + {[]byte("faa"), []byte("bra")}, } // and these should be not. - deletedKVs = []kvSeen{ - {[]byte("fee"), []byte("pow"), false}, - {[]byte("fii"), []byte("qaz"), false}, + deletedKVs = []KeyValue{ + {[]byte("fee"), []byte("pow")}, + {[]byte("fii"), []byte("qaz")}, } // and these should be not. - updatedKVs = []kvSeen{ - {[]byte("fuu"), []byte("wop"), false}, - {[]byte("fyy"), []byte("zaq"), false}, + updatedKVs = []KeyValue{ + {[]byte("fuu"), []byte("wop")}, + {[]byte("fyy"), []byte("zaq")}, } ps = NewMemoryStore() ts = NewMemCachedStore(ps) ) for _, v := range lowerKVs { - require.NoError(t, ps.Put(v.key, v.val)) + require.NoError(t, ps.Put(v.Key, v.Value)) } for _, v := range deletedKVs { - require.NoError(t, ps.Put(v.key, v.val)) - require.NoError(t, ts.Delete(v.key)) + require.NoError(t, ps.Put(v.Key, v.Value)) + require.NoError(t, ts.Delete(v.Key)) } for _, v := range updatedKVs { - require.NoError(t, ps.Put(v.key, []byte("stub"))) - require.NoError(t, ts.Put(v.key, v.val)) + require.NoError(t, ps.Put(v.Key, []byte("stub"))) + require.NoError(t, ts.Put(v.Key, v.Value)) } foundKVs := make(map[string][]byte) ts.Seek(goodPrefix, func(k, v []byte) { @@ -172,14 +172,14 @@ func TestCachedSeek(t *testing.T) { }) assert.Equal(t, len(foundKVs), len(lowerKVs)+len(updatedKVs)) for _, kv := range lowerKVs { - assert.Equal(t, kv.val, foundKVs[string(kv.key)]) + assert.Equal(t, kv.Value, foundKVs[string(kv.Key)]) } for _, kv := range deletedKVs { - _, ok := foundKVs[string(kv.key)] + _, ok := foundKVs[string(kv.Key)] assert.Equal(t, false, ok) } for _, kv := range updatedKVs { - assert.Equal(t, kv.val, foundKVs[string(kv.key)]) + assert.Equal(t, kv.Value, foundKVs[string(kv.Key)]) } } @@ -332,46 +332,46 @@ func TestCachedSeekSorting(t *testing.T) { // Given this prefix... goodPrefix = []byte{1} // these pairs should be found... - lowerKVs = []kvSeen{ - {[]byte{1, 2, 3}, []byte("bra"), false}, - {[]byte{1, 2, 5}, []byte("bar"), false}, - {[]byte{1, 3, 3}, []byte("bra"), false}, - {[]byte{1, 3, 5}, []byte("bra"), false}, + lowerKVs = []KeyValue{ + {[]byte{1, 2, 3}, []byte("bra")}, + {[]byte{1, 2, 5}, []byte("bar")}, + {[]byte{1, 3, 3}, []byte("bra")}, + {[]byte{1, 3, 5}, []byte("bra")}, } // and these should be not. - deletedKVs = []kvSeen{ - {[]byte{1, 7, 3}, []byte("pow"), false}, - {[]byte{1, 7, 4}, []byte("qaz"), false}, + deletedKVs = []KeyValue{ + {[]byte{1, 7, 3}, []byte("pow")}, + {[]byte{1, 7, 4}, []byte("qaz")}, } // and these should be not. - updatedKVs = []kvSeen{ - {[]byte{1, 2, 4}, []byte("zaq"), false}, - {[]byte{1, 2, 6}, []byte("zaq"), false}, - {[]byte{1, 3, 2}, []byte("wop"), false}, - {[]byte{1, 3, 4}, []byte("zaq"), false}, + updatedKVs = []KeyValue{ + {[]byte{1, 2, 4}, []byte("zaq")}, + {[]byte{1, 2, 6}, []byte("zaq")}, + {[]byte{1, 3, 2}, []byte("wop")}, + {[]byte{1, 3, 4}, []byte("zaq")}, } ps = NewMemoryStore() ts = NewMemCachedStore(ps) ) for _, v := range lowerKVs { - require.NoError(t, ps.Put(v.key, v.val)) + require.NoError(t, ps.Put(v.Key, v.Value)) } for _, v := range deletedKVs { - require.NoError(t, ps.Put(v.key, v.val)) - require.NoError(t, ts.Delete(v.key)) + require.NoError(t, ps.Put(v.Key, v.Value)) + require.NoError(t, ts.Delete(v.Key)) } for _, v := range updatedKVs { - require.NoError(t, ps.Put(v.key, []byte("stub"))) - require.NoError(t, ts.Put(v.key, v.val)) + require.NoError(t, ps.Put(v.Key, []byte("stub"))) + require.NoError(t, ts.Put(v.Key, v.Value)) } - var foundKVs []kvSeen + var foundKVs []KeyValue ts.Seek(goodPrefix, func(k, v []byte) { - foundKVs = append(foundKVs, kvSeen{key: slice.Copy(k), val: slice.Copy(v)}) + foundKVs = append(foundKVs, KeyValue{Key: slice.Copy(k), Value: slice.Copy(v)}) }) assert.Equal(t, len(foundKVs), len(lowerKVs)+len(updatedKVs)) expected := append(lowerKVs, updatedKVs...) sort.Slice(expected, func(i, j int) bool { - return bytes.Compare(expected[i].key, expected[j].key) < 0 + return bytes.Compare(expected[i].Key, expected[j].Key) < 0 }) require.Equal(t, expected, foundKVs) } diff --git a/pkg/core/storage/storeandbatch_test.go b/pkg/core/storage/storeandbatch_test.go index a8591ddfe..591afbcd2 100644 --- a/pkg/core/storage/storeandbatch_test.go +++ b/pkg/core/storage/storeandbatch_test.go @@ -1,8 +1,10 @@ package storage import ( + "bytes" "reflect" "runtime" + "sort" "testing" "github.com/nspcc-dev/neo-go/pkg/util/slice" @@ -10,13 +12,6 @@ import ( "github.com/stretchr/testify/require" ) -// kvSeen is used to test Seek implementations. -type kvSeen struct { - key []byte - val []byte - seen bool -} - type dbSetup struct { name string create func(testing.TB) Store @@ -76,47 +71,38 @@ func testStoreSeek(t *testing.T, s Store) { // Given this prefix... goodprefix = []byte{'f'} // these pairs should be found... - goodkvs = []kvSeen{ - {[]byte("foo"), []byte("bar"), false}, - {[]byte("faa"), []byte("bra"), false}, - {[]byte("foox"), []byte("barx"), false}, + goodkvs = []KeyValue{ + {[]byte("foo"), []byte("bar")}, + {[]byte("faa"), []byte("bra")}, + {[]byte("foox"), []byte("barx")}, } // and these should be not. - badkvs = []kvSeen{ - {[]byte("doo"), []byte("pow"), false}, - {[]byte("mew"), []byte("qaz"), false}, + badkvs = []KeyValue{ + {[]byte("doo"), []byte("pow")}, + {[]byte("mew"), []byte("qaz")}, } ) for _, v := range goodkvs { - require.NoError(t, s.Put(v.key, v.val)) + require.NoError(t, s.Put(v.Key, v.Value)) } for _, v := range badkvs { - require.NoError(t, s.Put(v.key, v.val)) + require.NoError(t, s.Put(v.Key, v.Value)) } - numFound := 0 - s.Seek(goodprefix, func(k, v []byte) { - for i := 0; i < len(goodkvs); i++ { - if string(k) == string(goodkvs[i].key) { - assert.Equal(t, string(goodkvs[i].val), string(v)) - goodkvs[i].seen = true - } - } - for i := 0; i < len(badkvs); i++ { - if string(k) == string(badkvs[i].key) { - badkvs[i].seen = true - } - } - numFound++ + // Seek result expected to be sorted in an ascending way. + sort.Slice(goodkvs, func(i, j int) bool { + return bytes.Compare(goodkvs[i].Key, goodkvs[j].Key) < 0 }) - assert.Equal(t, len(goodkvs), numFound) - for i := 0; i < len(goodkvs); i++ { - assert.Equal(t, true, goodkvs[i].seen) - } - for i := 0; i < len(badkvs); i++ { - assert.Equal(t, false, badkvs[i].seen) - } + + actual := make([]KeyValue, 0, len(goodkvs)) + s.Seek(goodprefix, func(k, v []byte) { + actual = append(actual, KeyValue{ + Key: slice.Copy(k), + Value: slice.Copy(v), + }) + }) + assert.Equal(t, goodkvs, actual) require.NoError(t, s.Close()) } From 7d5b20d8ddeefb464c325040dc0152d867400cd9 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Mon, 10 Jan 2022 16:55:57 +0300 Subject: [PATCH 3/6] core: fix comments formatting --- pkg/core/blockchain.go | 2 +- pkg/core/blockchain_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index d30ead62c..13adbefb4 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -2106,7 +2106,7 @@ func (bc *Blockchain) PoolTxWithData(t *transaction.Transaction, data interface{ return bc.verifyAndPoolTx(t, mp, feer, data) } -//GetStandByValidators returns validators from the configuration. +// GetStandByValidators returns validators from the configuration. func (bc *Blockchain) GetStandByValidators() keys.PublicKeys { return bc.sbCommittee[:bc.config.ValidatorsCount].Copy() } diff --git a/pkg/core/blockchain_test.go b/pkg/core/blockchain_test.go index 2e309189a..db78c8966 100644 --- a/pkg/core/blockchain_test.go +++ b/pkg/core/blockchain_test.go @@ -1051,7 +1051,7 @@ func TestVerifyTx(t *testing.T) { netFee, sizeDelta := fee.Calculate(bc.GetBaseExecFee(), testchain.MultisigVerificationScript()) tx.NetworkFee = netFee + // multisig witness verification price int64(size)*bc.FeePerByte() + // fee for unsigned size - int64(sizeDelta)*bc.FeePerByte() + //fee for multisig size + int64(sizeDelta)*bc.FeePerByte() + // fee for multisig size 66*bc.FeePerByte() + // fee for Notary signature size (66 bytes for Invocation script and 0 bytes for Verification script) 2*bc.FeePerByte() + // fee for the length of each script in Notary witness (they are nil, so we did not take them into account during `size` calculation) transaction.NotaryServiceFeePerKey + // fee for Notary attribute From 6bc92abe1986fbae4a68edbaea9be6ae2617ebf3 Mon Sep 17 00:00:00 2001 From: AnnaShaleva Date: Thu, 16 Dec 2021 16:55:50 +0300 Subject: [PATCH 4/6] storage: allow to seek starting from some point --- pkg/core/blockchain.go | 2 +- pkg/core/blockchain_test.go | 2 +- pkg/core/dao/dao.go | 40 ++++---- pkg/core/interop_system.go | 2 +- pkg/core/native/management.go | 2 +- pkg/core/native/native_neo.go | 4 +- pkg/core/stateroot/module.go | 2 +- pkg/core/statesync/module_test.go | 4 +- pkg/core/statesync_test.go | 4 +- pkg/core/storage/boltdb_store.go | 10 +- pkg/core/storage/leveldb_store.go | 9 +- pkg/core/storage/memcached_store.go | 38 ++++--- pkg/core/storage/memcached_store_test.go | 8 +- pkg/core/storage/memory_store.go | 20 ++-- pkg/core/storage/memory_store_test.go | 2 +- pkg/core/storage/store.go | 18 +++- pkg/core/storage/storeandbatch_test.go | 120 +++++++++++++++++------ 17 files changed, 196 insertions(+), 91 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 13adbefb4..73d923917 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -486,7 +486,7 @@ func (bc *Blockchain) removeOldStorageItems() { b := bc.dao.Store.Batch() prefix := statesync.TemporaryPrefix(bc.dao.Version.StoragePrefix) - bc.dao.Store.Seek([]byte{byte(prefix)}, func(k, _ []byte) { + bc.dao.Store.Seek(storage.SeekRange{Prefix: []byte{byte(prefix)}}, func(k, _ []byte) { // #1468, but don't need to copy here, because it is done by Store. b.Delete(k) }) diff --git a/pkg/core/blockchain_test.go b/pkg/core/blockchain_test.go index db78c8966..ac146c4ac 100644 --- a/pkg/core/blockchain_test.go +++ b/pkg/core/blockchain_test.go @@ -1769,7 +1769,7 @@ func TestBlockchain_InitWithIncompleteStateJump(t *testing.T) { if bcSpout.dao.Version.StoragePrefix == tempPrefix { tempPrefix = storage.STStorage } - bcSpout.dao.Store.Seek(bcSpout.dao.Version.StoragePrefix.Bytes(), func(k, v []byte) { + bcSpout.dao.Store.Seek(storage.SeekRange{Prefix: bcSpout.dao.Version.StoragePrefix.Bytes()}, func(k, v []byte) { key := slice.Copy(k) key[0] = byte(tempPrefix) value := slice.Copy(v) diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index 4220f6e0e..c137dd434 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -62,8 +62,8 @@ type DAO interface { PutStateSyncCurrentBlockHeight(h uint32) error PutStorageItem(id int32, key []byte, si state.StorageItem) error PutVersion(v Version) error - Seek(id int32, prefix []byte, f func(k, v []byte)) - SeekAsync(ctx context.Context, id int32, prefix []byte) chan storage.KeyValue + Seek(id int32, rng storage.SeekRange, f func(k, v []byte)) + SeekAsync(ctx context.Context, id int32, rng storage.SeekRange) chan storage.KeyValue StoreAsBlock(block *block.Block, aer1 *state.AppExecResult, aer2 *state.AppExecResult, buf *io.BufBinWriter) error StoreAsCurrentBlock(block *block.Block, buf *io.BufBinWriter) error StoreAsTransaction(tx *transaction.Transaction, index uint32, aer *state.AppExecResult, buf *io.BufBinWriter) error @@ -300,30 +300,26 @@ func (dao *Simple) GetStorageItemsWithPrefix(id int32, prefix []byte) ([]state.S Item: state.StorageItem(v), }) } - dao.Seek(id, prefix, saveToArr) + dao.Seek(id, storage.SeekRange{Prefix: prefix}, saveToArr) return siArr, nil } -// Seek executes f for all items with a given prefix. -// If key is to be used outside of f, they may not be copied. -func (dao *Simple) Seek(id int32, prefix []byte, f func(k, v []byte)) { - lookupKey := makeStorageItemKey(dao.Version.StoragePrefix, id, nil) - if prefix != nil { - lookupKey = append(lookupKey, prefix...) - } - dao.Store.Seek(lookupKey, func(k, v []byte) { - f(k[len(lookupKey):], v) +// Seek executes f for all storage items matching a given `rng` (matching given prefix and +// starting from the point specified). If key or value is to be used outside of f, they +// may not be copied. +func (dao *Simple) Seek(id int32, rng storage.SeekRange, f func(k, v []byte)) { + rng.Prefix = makeStorageItemKey(dao.Version.StoragePrefix, id, rng.Prefix) + dao.Store.Seek(rng, func(k, v []byte) { + f(k[len(rng.Prefix):], v) }) } -// SeekAsync sends all storage items matching given prefix to a channel and returns -// the channel. Resulting keys and values may not be copied. -func (dao *Simple) SeekAsync(ctx context.Context, id int32, prefix []byte) chan storage.KeyValue { - lookupKey := makeStorageItemKey(dao.Version.StoragePrefix, id, nil) - if prefix != nil { - lookupKey = append(lookupKey, prefix...) - } - return dao.Store.SeekAsync(ctx, lookupKey, true) +// SeekAsync sends all storage items matching a given `rng` (matching given prefix and +// starting from the point specified) to a channel and returns the channel. +// Resulting keys and values may not be copied. +func (dao *Simple) SeekAsync(ctx context.Context, id int32, rng storage.SeekRange) chan storage.KeyValue { + rng.Prefix = makeStorageItemKey(dao.Version.StoragePrefix, id, rng.Prefix) + return dao.Store.SeekAsync(ctx, rng, true) } // makeStorageItemKey returns a key used to store StorageItem in the DB. @@ -479,7 +475,9 @@ func (dao *Simple) GetStateSyncCurrentBlockHeight() (uint32, error) { // the given underlying store. func (dao *Simple) GetHeaderHashes() ([]util.Uint256, error) { hashMap := make(map[uint32][]util.Uint256) - dao.Store.Seek(storage.IXHeaderHashList.Bytes(), func(k, v []byte) { + dao.Store.Seek(storage.SeekRange{ + Prefix: storage.IXHeaderHashList.Bytes(), + }, func(k, v []byte) { storedCount := binary.LittleEndian.Uint32(k[1:]) hashes, err := read2000Uint256Hashes(v) if err != nil { diff --git a/pkg/core/interop_system.go b/pkg/core/interop_system.go index fd473b65b..7908e17d4 100644 --- a/pkg/core/interop_system.go +++ b/pkg/core/interop_system.go @@ -190,7 +190,7 @@ func storageFind(ic *interop.Context) error { // Items in seekres should be sorted by key, but GetStorageItemsWithPrefix returns // sorted items, so no need to sort them one more time. ctx, cancel := context.WithCancel(context.Background()) - seekres := ic.DAO.SeekAsync(ctx, stc.ID, prefix) + seekres := ic.DAO.SeekAsync(ctx, stc.ID, storage.SeekRange{Prefix: prefix}) item := istorage.NewIterator(seekres, prefix, opts) ic.VM.Estack().PushItem(stackitem.NewInterop(item)) ic.RegisterCancelFunc(cancel) diff --git a/pkg/core/native/management.go b/pkg/core/native/management.go index 3b63cfef0..2180f7aee 100644 --- a/pkg/core/native/management.go +++ b/pkg/core/native/management.go @@ -503,7 +503,7 @@ func (m *Management) InitializeCache(d dao.DAO) error { defer m.mtx.Unlock() var initErr error - d.Seek(m.ID, []byte{prefixContract}, func(_, v []byte) { + d.Seek(m.ID, storage.SeekRange{Prefix: []byte{prefixContract}}, func(_, v []byte) { var cs = new(state.Contract) initErr = stackitem.DeserializeConvertible(v, cs) if initErr != nil { diff --git a/pkg/core/native/native_neo.go b/pkg/core/native/native_neo.go index 163525080..b90091909 100644 --- a/pkg/core/native/native_neo.go +++ b/pkg/core/native/native_neo.go @@ -386,7 +386,7 @@ func (n *NEO) PostPersist(ic *interop.Context) error { func (n *NEO) getGASPerVote(d dao.DAO, key []byte, index ...uint32) []big.Int { var max = make([]uint32, len(index)) var reward = make([]big.Int, len(index)) - d.Seek(n.ID, key, func(k, v []byte) { + d.Seek(n.ID, storage.SeekRange{Prefix: key}, func(k, v []byte) { if len(k) == 4 { num := binary.BigEndian.Uint32(k) for i, ind := range index { @@ -591,7 +591,7 @@ func (n *NEO) dropCandidateIfZero(d dao.DAO, pub *keys.PublicKey, c *candidate) var toRemove []string voterKey := makeVoterKey(pub.Bytes()) - d.Seek(n.ID, voterKey, func(k, v []byte) { + d.Seek(n.ID, storage.SeekRange{Prefix: voterKey}, func(k, v []byte) { toRemove = append(toRemove, string(k)) }) for i := range toRemove { diff --git a/pkg/core/stateroot/module.go b/pkg/core/stateroot/module.go index 322e23c8e..86d3e7c54 100644 --- a/pkg/core/stateroot/module.go +++ b/pkg/core/stateroot/module.go @@ -127,7 +127,7 @@ func (s *Module) CleanStorage() error { return fmt.Errorf("can't clean MPT data for non-genesis block: expected local stateroot height 0, got %d", s.localHeight.Load()) } b := s.Store.Batch() - s.Store.Seek([]byte{byte(storage.DataMPT)}, func(k, _ []byte) { + s.Store.Seek(storage.SeekRange{Prefix: []byte{byte(storage.DataMPT)}}, func(k, _ []byte) { // #1468, but don't need to copy here, because it is done by Store. b.Delete(k) }) diff --git a/pkg/core/statesync/module_test.go b/pkg/core/statesync/module_test.go index 538a273eb..ab4ba650e 100644 --- a/pkg/core/statesync/module_test.go +++ b/pkg/core/statesync/module_test.go @@ -32,7 +32,7 @@ func TestModule_PR2019_discussion_r689629704(t *testing.T) { nodes = make(map[util.Uint256][]byte) expectedItems []storage.KeyValue ) - expectedStorage.Seek(storage.DataMPT.Bytes(), func(k, v []byte) { + expectedStorage.Seek(storage.SeekRange{Prefix: storage.DataMPT.Bytes()}, func(k, v []byte) { key := slice.Copy(k) value := slice.Copy(v) expectedItems = append(expectedItems, storage.KeyValue{ @@ -95,7 +95,7 @@ func TestModule_PR2019_discussion_r689629704(t *testing.T) { // Compare resulting storage items and refcounts. var actualItems []storage.KeyValue - expectedStorage.Seek(storage.DataMPT.Bytes(), func(k, v []byte) { + expectedStorage.Seek(storage.SeekRange{Prefix: storage.DataMPT.Bytes()}, func(k, v []byte) { key := slice.Copy(k) value := slice.Copy(v) actualItems = append(actualItems, storage.KeyValue{ diff --git a/pkg/core/statesync_test.go b/pkg/core/statesync_test.go index 3b802b757..9996579f2 100644 --- a/pkg/core/statesync_test.go +++ b/pkg/core/statesync_test.go @@ -424,7 +424,7 @@ func TestStateSyncModule_RestoreBasicChain(t *testing.T) { // compare storage states fetchStorage := func(bc *Blockchain) []storage.KeyValue { var kv []storage.KeyValue - bc.dao.Store.Seek(bc.dao.Version.StoragePrefix.Bytes(), func(k, v []byte) { + bc.dao.Store.Seek(storage.SeekRange{Prefix: bc.dao.Version.StoragePrefix.Bytes()}, func(k, v []byte) { key := slice.Copy(k) value := slice.Copy(v) if key[0] == byte(storage.STTempStorage) { @@ -444,7 +444,7 @@ func TestStateSyncModule_RestoreBasicChain(t *testing.T) { // no temp items should be left require.Eventually(t, func() bool { var haveItems bool - bcBolt.dao.Store.Seek(storage.STStorage.Bytes(), func(_, _ []byte) { + bcBolt.dao.Store.Seek(storage.SeekRange{Prefix: storage.STStorage.Bytes()}, func(_, _ []byte) { haveItems = true }) return !haveItems diff --git a/pkg/core/storage/boltdb_store.go b/pkg/core/storage/boltdb_store.go index 4112ed950..540bc84fd 100644 --- a/pkg/core/storage/boltdb_store.go +++ b/pkg/core/storage/boltdb_store.go @@ -109,11 +109,15 @@ func (s *BoltDBStore) PutChangeSet(puts map[string][]byte, dels map[string]bool) } // Seek implements the Store interface. -func (s *BoltDBStore) Seek(key []byte, f func(k, v []byte)) { +func (s *BoltDBStore) Seek(rng SeekRange, f func(k, v []byte)) { + start := make([]byte, len(rng.Prefix)+len(rng.Start)) + copy(start, rng.Prefix) + copy(start[len(rng.Prefix):], rng.Start) + prefix := util.BytesPrefix(rng.Prefix) + prefix.Start = start err := s.db.View(func(tx *bbolt.Tx) error { c := tx.Bucket(Bucket).Cursor() - prefix := util.BytesPrefix(key) - for k, v := c.Seek(prefix.Start); k != nil && bytes.Compare(k, prefix.Limit) <= 0; k, v = c.Next() { + for k, v := c.Seek(prefix.Start); k != nil && (len(prefix.Limit) == 0 || bytes.Compare(k, prefix.Limit) <= 0); k, v = c.Next() { f(k, v) } return nil diff --git a/pkg/core/storage/leveldb_store.go b/pkg/core/storage/leveldb_store.go index e348c3c2e..902041d50 100644 --- a/pkg/core/storage/leveldb_store.go +++ b/pkg/core/storage/leveldb_store.go @@ -85,8 +85,13 @@ func (s *LevelDBStore) PutChangeSet(puts map[string][]byte, dels map[string]bool } // Seek implements the Store interface. -func (s *LevelDBStore) Seek(key []byte, f func(k, v []byte)) { - iter := s.db.NewIterator(util.BytesPrefix(key), nil) +func (s *LevelDBStore) Seek(rng SeekRange, f func(k, v []byte)) { + start := make([]byte, len(rng.Prefix)+len(rng.Start)) + copy(start, rng.Prefix) + copy(start[len(rng.Prefix):], rng.Start) + prefix := util.BytesPrefix(rng.Prefix) + prefix.Start = start + iter := s.db.NewIterator(prefix, nil) for iter.Next() { f(iter.Key(), iter.Value()) } diff --git a/pkg/core/storage/memcached_store.go b/pkg/core/storage/memcached_store.go index d9a6dcbbc..e9f2bd76d 100644 --- a/pkg/core/storage/memcached_store.go +++ b/pkg/core/storage/memcached_store.go @@ -90,17 +90,17 @@ func (s *MemCachedStore) GetBatch() *MemBatch { } // Seek implements the Store interface. -func (s *MemCachedStore) Seek(key []byte, f func(k, v []byte)) { - s.seek(context.Background(), key, false, f) +func (s *MemCachedStore) Seek(rng SeekRange, f func(k, v []byte)) { + s.seek(context.Background(), rng, false, f) } // SeekAsync returns non-buffered channel with matching KeyValue pairs. Key and // value slices may not be copied and may be modified. SeekAsync can guarantee // that key-value items are sorted by key in ascending way. -func (s *MemCachedStore) SeekAsync(ctx context.Context, key []byte, cutPrefix bool) chan KeyValue { +func (s *MemCachedStore) SeekAsync(ctx context.Context, rng SeekRange, cutPrefix bool) chan KeyValue { res := make(chan KeyValue) go func() { - s.seek(ctx, key, cutPrefix, func(k, v []byte) { + s.seek(ctx, rng, cutPrefix, func(k, v []byte) { res <- KeyValue{ Key: k, Value: v, @@ -112,13 +112,23 @@ func (s *MemCachedStore) SeekAsync(ctx context.Context, key []byte, cutPrefix bo return res } -func (s *MemCachedStore) seek(ctx context.Context, key []byte, cutPrefix bool, f func(k, v []byte)) { +// seek is internal representations of Seek* capable of seeking for the given key +// and supporting early stop using provided context. `cutPrefix` denotes whether provided +// key needs to be cut off the resulting keys. `rng` specifies prefix items must match +// and point to start seeking from. +func (s *MemCachedStore) seek(ctx context.Context, rng SeekRange, cutPrefix bool, f func(k, v []byte)) { // Create memory store `mem` and `del` snapshot not to hold the lock. var memRes []KeyValueExists - sk := string(key) + sPrefix := string(rng.Prefix) + lPrefix := len(sPrefix) + sStart := string(rng.Start) + lStart := len(sStart) + isKeyOK := func(key string) bool { + return strings.HasPrefix(key, sPrefix) && (lStart == 0 || strings.Compare(key[lPrefix:], sStart) >= 0) + } s.mut.RLock() for k, v := range s.MemoryStore.mem { - if strings.HasPrefix(k, sk) { + if isKeyOK(k) { memRes = append(memRes, KeyValueExists{ KeyValue: KeyValue{ Key: []byte(k), @@ -129,7 +139,7 @@ func (s *MemCachedStore) seek(ctx context.Context, key []byte, cutPrefix bool, f } } for k := range s.MemoryStore.del { - if strings.HasPrefix(k, sk) { + if isKeyOK(k) { memRes = append(memRes, KeyValueExists{ KeyValue: KeyValue{ Key: []byte(k), @@ -156,7 +166,7 @@ func (s *MemCachedStore) seek(ctx context.Context, key []byte, cutPrefix bool, f iMem++ } // Merge results of seek operations in ascending order. - ps.Seek(key, func(k, v []byte) { + mergeFunc := func(k, v []byte) { if done { return } @@ -175,7 +185,7 @@ func (s *MemCachedStore) seek(ctx context.Context, key []byte, cutPrefix bool, f if isMem { if kvMem.Exists { if cutPrefix { - kvMem.Key = kvMem.Key[len(key):] + kvMem.Key = kvMem.Key[lPrefix:] } f(kvMem.Key, kvMem.Value) } @@ -189,7 +199,7 @@ func (s *MemCachedStore) seek(ctx context.Context, key []byte, cutPrefix bool, f } else { if !bytes.Equal(kvMem.Key, kvPs.Key) { if cutPrefix { - kvPs.Key = kvPs.Key[len(key):] + kvPs.Key = kvPs.Key[lPrefix:] } f(kvPs.Key, kvPs.Value) } @@ -197,7 +207,9 @@ func (s *MemCachedStore) seek(ctx context.Context, key []byte, cutPrefix bool, f } } } - }) + } + ps.Seek(rng, mergeFunc) + if !done && haveMem { loop: for i := iMem - 1; i < len(memRes); i++ { @@ -208,7 +220,7 @@ func (s *MemCachedStore) seek(ctx context.Context, key []byte, cutPrefix bool, f kvMem = memRes[i] if kvMem.Exists { if cutPrefix { - kvMem.Key = kvMem.Key[len(key):] + kvMem.Key = kvMem.Key[lPrefix:] } f(kvMem.Key, kvMem.Value) } diff --git a/pkg/core/storage/memcached_store_test.go b/pkg/core/storage/memcached_store_test.go index f5c380238..44582e507 100644 --- a/pkg/core/storage/memcached_store_test.go +++ b/pkg/core/storage/memcached_store_test.go @@ -167,7 +167,7 @@ func TestCachedSeek(t *testing.T) { require.NoError(t, ts.Put(v.Key, v.Value)) } foundKVs := make(map[string][]byte) - ts.Seek(goodPrefix, func(k, v []byte) { + ts.Seek(SeekRange{Prefix: goodPrefix}, func(k, v []byte) { foundKVs[string(k)] = v }) assert.Equal(t, len(foundKVs), len(lowerKVs)+len(updatedKVs)) @@ -232,7 +232,7 @@ func benchmarkCachedSeek(t *testing.B, ps Store, psElementsCount, tsElementsCoun t.ReportAllocs() t.ResetTimer() for n := 0; n < t.N; n++ { - ts.Seek(searchPrefix, func(k, v []byte) {}) + ts.Seek(SeekRange{Prefix: searchPrefix}, func(k, v []byte) {}) } t.StopTimer() } @@ -290,7 +290,7 @@ func (b *BadStore) PutChangeSet(_ map[string][]byte, _ map[string]bool) error { b.onPutBatch() return ErrKeyNotFound } -func (b *BadStore) Seek(k []byte, f func(k, v []byte)) { +func (b *BadStore) Seek(rng SeekRange, f func(k, v []byte)) { } func (b *BadStore) Close() error { return nil @@ -365,7 +365,7 @@ func TestCachedSeekSorting(t *testing.T) { require.NoError(t, ts.Put(v.Key, v.Value)) } var foundKVs []KeyValue - ts.Seek(goodPrefix, func(k, v []byte) { + ts.Seek(SeekRange{Prefix: goodPrefix}, func(k, v []byte) { foundKVs = append(foundKVs, KeyValue{Key: slice.Copy(k), Value: slice.Copy(v)}) }) assert.Equal(t, len(foundKVs), len(lowerKVs)+len(updatedKVs)) diff --git a/pkg/core/storage/memory_store.go b/pkg/core/storage/memory_store.go index 21ca47645..2f0319b5d 100644 --- a/pkg/core/storage/memory_store.go +++ b/pkg/core/storage/memory_store.go @@ -104,9 +104,9 @@ func (s *MemoryStore) PutChangeSet(puts map[string][]byte, dels map[string]bool) } // Seek implements the Store interface. -func (s *MemoryStore) Seek(key []byte, f func(k, v []byte)) { +func (s *MemoryStore) Seek(rng SeekRange, f func(k, v []byte)) { s.mut.RLock() - s.seek(key, f) + s.seek(rng, f) s.mut.RUnlock() } @@ -127,12 +127,20 @@ func (s *MemoryStore) SeekAll(key []byte, f func(k, v []byte)) { } } -// seek is an internal unlocked implementation of Seek. -func (s *MemoryStore) seek(key []byte, f func(k, v []byte)) { - sk := string(key) +// seek is an internal unlocked implementation of Seek. `start` denotes whether +// seeking starting from the provided prefix should be performed. +func (s *MemoryStore) seek(rng SeekRange, f func(k, v []byte)) { + sPrefix := string(rng.Prefix) + lPrefix := len(sPrefix) + sStart := string(rng.Start) + lStart := len(sStart) var memList []KeyValue + + isKeyOK := func(key string) bool { + return strings.HasPrefix(key, sPrefix) && (lStart == 0 || strings.Compare(key[lPrefix:], sStart) >= 0) + } for k, v := range s.mem { - if strings.HasPrefix(k, sk) { + if isKeyOK(k) { memList = append(memList, KeyValue{ Key: []byte(k), Value: v, diff --git a/pkg/core/storage/memory_store_test.go b/pkg/core/storage/memory_store_test.go index 6bb7d7526..4069cd37d 100644 --- a/pkg/core/storage/memory_store_test.go +++ b/pkg/core/storage/memory_store_test.go @@ -28,7 +28,7 @@ func BenchmarkMemorySeek(t *testing.B) { t.ReportAllocs() t.ResetTimer() for n := 0; n < t.N; n++ { - ms.Seek(searchPrefix, func(k, v []byte) {}) + ms.Seek(SeekRange{Prefix: searchPrefix}, func(k, v []byte) {}) } }) } diff --git a/pkg/core/storage/store.go b/pkg/core/storage/store.go index 40014e0c0..6b85903bf 100644 --- a/pkg/core/storage/store.go +++ b/pkg/core/storage/store.go @@ -45,6 +45,22 @@ const ( MaxStorageValueLen = 65535 ) +// SeekRange represents options for Store.Seek operation. +type SeekRange struct { + // Prefix denotes the Seek's lookup key. + // Empty Prefix means seeking through all keys in the DB starting from + // the Start if specified. + Prefix []byte + // Start denotes value upended to the Prefix to start Seek from. + // Seeking starting from some key includes this key to the result; + // if no matching key was found then next suitable key is picked up. + // Start may be empty. Empty Start means seeking through all keys in + // the DB with matching Prefix. + // Empty Prefix and empty Start can be combined, which means seeking + // through all keys in the DB. + Start []byte +} + // ErrKeyNotFound is an error returned by Store implementations // when a certain key is not found. var ErrKeyNotFound = errors.New("key not found") @@ -63,7 +79,7 @@ type ( // Seek can guarantee that provided key (k) and value (v) are the only valid until the next call to f. // Key and value slices should not be modified. Seek can guarantee that key-value items are sorted by // key in ascending way. - Seek(k []byte, f func(k, v []byte)) + Seek(rng SeekRange, f func(k, v []byte)) Close() error } diff --git a/pkg/core/storage/storeandbatch_test.go b/pkg/core/storage/storeandbatch_test.go index 591afbcd2..9423db635 100644 --- a/pkg/core/storage/storeandbatch_test.go +++ b/pkg/core/storage/storeandbatch_test.go @@ -67,42 +67,104 @@ func testStorePutBatch(t *testing.T, s Store) { } func testStoreSeek(t *testing.T, s Store) { - var ( - // Given this prefix... - goodprefix = []byte{'f'} - // these pairs should be found... - goodkvs = []KeyValue{ - {[]byte("foo"), []byte("bar")}, - {[]byte("faa"), []byte("bra")}, - {[]byte("foox"), []byte("barx")}, - } - // and these should be not. - badkvs = []KeyValue{ - {[]byte("doo"), []byte("pow")}, - {[]byte("mew"), []byte("qaz")}, - } - ) - - for _, v := range goodkvs { - require.NoError(t, s.Put(v.Key, v.Value)) + // Use the same set of kvs to test Seek with different prefix/start values. + kvs := []KeyValue{ + {[]byte("10"), []byte("bar")}, + {[]byte("11"), []byte("bara")}, + {[]byte("20"), []byte("barb")}, + {[]byte("21"), []byte("barc")}, + {[]byte("22"), []byte("bard")}, + {[]byte("30"), []byte("bare")}, + {[]byte("31"), []byte("barf")}, } - for _, v := range badkvs { + for _, v := range kvs { require.NoError(t, s.Put(v.Key, v.Value)) } - // Seek result expected to be sorted in an ascending way. - sort.Slice(goodkvs, func(i, j int) bool { - return bytes.Compare(goodkvs[i].Key, goodkvs[j].Key) < 0 - }) + check := func(t *testing.T, goodprefix, start []byte, goodkvs []KeyValue) { + // Seek result expected to be sorted in an ascending way. + sort.Slice(goodkvs, func(i, j int) bool { + return bytes.Compare(goodkvs[i].Key, goodkvs[j].Key) < 0 + }) - actual := make([]KeyValue, 0, len(goodkvs)) - s.Seek(goodprefix, func(k, v []byte) { - actual = append(actual, KeyValue{ - Key: slice.Copy(k), - Value: slice.Copy(v), + actual := make([]KeyValue, 0, len(goodkvs)) + s.Seek(SeekRange{ + Prefix: goodprefix, + Start: start, + }, func(k, v []byte) { + actual = append(actual, KeyValue{ + Key: slice.Copy(k), + Value: slice.Copy(v), + }) + }) + assert.Equal(t, goodkvs, actual) + } + + t.Run("non-empty prefix, empty start", func(t *testing.T) { + t.Run("good", func(t *testing.T) { + // Given this prefix... + goodprefix := []byte("2") + // and empty start range... + start := []byte{} + // these pairs should be found. + goodkvs := []KeyValue{ + kvs[2], // key = "20" + kvs[3], // key = "21" + kvs[4], // key = "22" + } + check(t, goodprefix, start, goodkvs) + }) + t.Run("no matching items", func(t *testing.T) { + goodprefix := []byte("0") + start := []byte{} + check(t, goodprefix, start, []KeyValue{}) }) }) - assert.Equal(t, goodkvs, actual) + + t.Run("non-empty prefix, non-empty start", func(t *testing.T) { + t.Run("good", func(t *testing.T) { + goodprefix := []byte("2") + start := []byte("1") // start will be upended to goodprefix to start seek from + goodkvs := []KeyValue{ + kvs[3], // key = "21" + kvs[4], // key = "22" + } + check(t, goodprefix, start, goodkvs) + }) + t.Run("no matching items", func(t *testing.T) { + goodprefix := []byte("2") + start := []byte("3") // start will be upended to goodprefix to start seek from + check(t, goodprefix, start, []KeyValue{}) + }) + }) + + t.Run("empty prefix, non-empty start", func(t *testing.T) { + t.Run("good", func(t *testing.T) { + goodprefix := []byte{} + start := []byte("21") + goodkvs := []KeyValue{ + kvs[3], // key = "21" + kvs[4], // key = "22" + kvs[5], // key = "30" + kvs[6], // key = "31" + } + check(t, goodprefix, start, goodkvs) + }) + t.Run("no matching items", func(t *testing.T) { + goodprefix := []byte{} + start := []byte("32") + check(t, goodprefix, start, []KeyValue{}) + }) + }) + + t.Run("empty prefix, empty start", func(t *testing.T) { + goodprefix := []byte{} + start := []byte{} + goodkvs := make([]KeyValue, len(kvs)) + copy(goodkvs, kvs) + check(t, goodprefix, start, goodkvs) + }) + require.NoError(t, s.Close()) } From 04a8e6666ff50722a07e172ef0b2ff2706ea05dc Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 28 Dec 2021 16:01:44 +0300 Subject: [PATCH 5/6] storage: allow to seek backwards --- pkg/core/storage/boltdb_store.go | 30 ++++- pkg/core/storage/leveldb_store.go | 21 ++- pkg/core/storage/memcached_store.go | 17 ++- pkg/core/storage/memory_store.go | 15 ++- pkg/core/storage/store.go | 4 + pkg/core/storage/storeandbatch_test.go | 173 ++++++++++++++++++------- 6 files changed, 203 insertions(+), 57 deletions(-) diff --git a/pkg/core/storage/boltdb_store.go b/pkg/core/storage/boltdb_store.go index 540bc84fd..c1ac2ed5e 100644 --- a/pkg/core/storage/boltdb_store.go +++ b/pkg/core/storage/boltdb_store.go @@ -113,7 +113,15 @@ func (s *BoltDBStore) Seek(rng SeekRange, f func(k, v []byte)) { start := make([]byte, len(rng.Prefix)+len(rng.Start)) copy(start, rng.Prefix) copy(start[len(rng.Prefix):], rng.Start) - prefix := util.BytesPrefix(rng.Prefix) + if rng.Backwards { + s.seekBackwards(rng.Prefix, start, f) + } else { + s.seek(rng.Prefix, start, f) + } +} + +func (s *BoltDBStore) seek(key []byte, start []byte, f func(k, v []byte)) { + prefix := util.BytesPrefix(key) prefix.Start = start err := s.db.View(func(tx *bbolt.Tx) error { c := tx.Bucket(Bucket).Cursor() @@ -127,6 +135,26 @@ func (s *BoltDBStore) Seek(rng SeekRange, f func(k, v []byte)) { } } +func (s *BoltDBStore) seekBackwards(key []byte, start []byte, f func(k, v []byte)) { + err := s.db.View(func(tx *bbolt.Tx) error { + c := tx.Bucket(Bucket).Cursor() + // Move cursor to the first kv pair which is followed by the pair matching the specified prefix. + if len(start) == 0 { + lastKey, _ := c.Last() + start = lastKey + } + rng := util.BytesPrefix(start) // in fact, we only need limit based on start slice to iterate backwards starting from this limit + c.Seek(rng.Limit) + for k, v := c.Prev(); k != nil && bytes.HasPrefix(k, key); k, v = c.Prev() { + f(k, v) + } + return nil + }) + if err != nil { + panic(err) + } +} + // Batch implements the Batch interface and returns a boltdb // compatible Batch. func (s *BoltDBStore) Batch() Batch { diff --git a/pkg/core/storage/leveldb_store.go b/pkg/core/storage/leveldb_store.go index 902041d50..87ab09923 100644 --- a/pkg/core/storage/leveldb_store.go +++ b/pkg/core/storage/leveldb_store.go @@ -89,7 +89,15 @@ func (s *LevelDBStore) Seek(rng SeekRange, f func(k, v []byte)) { start := make([]byte, len(rng.Prefix)+len(rng.Start)) copy(start, rng.Prefix) copy(start[len(rng.Prefix):], rng.Start) - prefix := util.BytesPrefix(rng.Prefix) + if rng.Backwards { + s.seekBackwards(rng.Prefix, start, f) + } else { + s.seek(rng.Prefix, start, f) + } +} + +func (s *LevelDBStore) seek(key []byte, start []byte, f func(k, v []byte)) { + prefix := util.BytesPrefix(key) prefix.Start = start iter := s.db.NewIterator(prefix, nil) for iter.Next() { @@ -98,6 +106,17 @@ func (s *LevelDBStore) Seek(rng SeekRange, f func(k, v []byte)) { iter.Release() } +func (s *LevelDBStore) seekBackwards(key []byte, start []byte, f func(k, v []byte)) { + iRange := util.BytesPrefix(start) + iRange.Start = key + + iter := s.db.NewIterator(iRange, nil) + for ok := iter.Last(); ok; ok = iter.Prev() { + f(iter.Key(), iter.Value()) + } + iter.Release() +} + // Batch implements the Batch interface and returns a leveldb // compatible Batch. func (s *LevelDBStore) Batch() Batch { diff --git a/pkg/core/storage/memcached_store.go b/pkg/core/storage/memcached_store.go index e9f2bd76d..7289e5f3c 100644 --- a/pkg/core/storage/memcached_store.go +++ b/pkg/core/storage/memcached_store.go @@ -115,7 +115,8 @@ func (s *MemCachedStore) SeekAsync(ctx context.Context, rng SeekRange, cutPrefix // seek is internal representations of Seek* capable of seeking for the given key // and supporting early stop using provided context. `cutPrefix` denotes whether provided // key needs to be cut off the resulting keys. `rng` specifies prefix items must match -// and point to start seeking from. +// and point to start seeking from. Backwards seeking from some point is supported +// with corresponding `rng` field set. func (s *MemCachedStore) seek(ctx context.Context, rng SeekRange, cutPrefix bool, f func(k, v []byte)) { // Create memory store `mem` and `del` snapshot not to hold the lock. var memRes []KeyValueExists @@ -126,6 +127,11 @@ func (s *MemCachedStore) seek(ctx context.Context, rng SeekRange, cutPrefix bool isKeyOK := func(key string) bool { return strings.HasPrefix(key, sPrefix) && (lStart == 0 || strings.Compare(key[lPrefix:], sStart) >= 0) } + if rng.Backwards { + isKeyOK = func(key string) bool { + return strings.HasPrefix(key, sPrefix) && (lStart == 0 || strings.Compare(key[lPrefix:], sStart) <= 0) + } + } s.mut.RLock() for k, v := range s.MemoryStore.mem { if isKeyOK(k) { @@ -149,9 +155,14 @@ func (s *MemCachedStore) seek(ctx context.Context, rng SeekRange, cutPrefix bool } ps := s.ps s.mut.RUnlock() + + less := func(k1, k2 []byte) bool { + res := bytes.Compare(k1, k2) + return res != 0 && rng.Backwards == (res > 0) + } // Sort memRes items for further comparison with ps items. sort.Slice(memRes, func(i, j int) bool { - return bytes.Compare(memRes[i].Key, memRes[j].Key) < 0 + return less(memRes[i].Key, memRes[j].Key) }) var ( @@ -181,7 +192,7 @@ func (s *MemCachedStore) seek(ctx context.Context, rng SeekRange, cutPrefix bool done = true break loop default: - var isMem = haveMem && (bytes.Compare(kvMem.Key, kvPs.Key) < 0) + var isMem = haveMem && less(kvMem.Key, kvPs.Key) if isMem { if kvMem.Exists { if cutPrefix { diff --git a/pkg/core/storage/memory_store.go b/pkg/core/storage/memory_store.go index 2f0319b5d..edcb6eb30 100644 --- a/pkg/core/storage/memory_store.go +++ b/pkg/core/storage/memory_store.go @@ -128,7 +128,8 @@ func (s *MemoryStore) SeekAll(key []byte, f func(k, v []byte)) { } // seek is an internal unlocked implementation of Seek. `start` denotes whether -// seeking starting from the provided prefix should be performed. +// seeking starting from the provided prefix should be performed. Backwards +// seeking from some point is supported with corresponding SeekRange field set. func (s *MemoryStore) seek(rng SeekRange, f func(k, v []byte)) { sPrefix := string(rng.Prefix) lPrefix := len(sPrefix) @@ -139,6 +140,16 @@ func (s *MemoryStore) seek(rng SeekRange, f func(k, v []byte)) { isKeyOK := func(key string) bool { return strings.HasPrefix(key, sPrefix) && (lStart == 0 || strings.Compare(key[lPrefix:], sStart) >= 0) } + if rng.Backwards { + isKeyOK = func(key string) bool { + return strings.HasPrefix(key, sPrefix) && (lStart == 0 || strings.Compare(key[lPrefix:], sStart) <= 0) + } + } + less := func(k1, k2 []byte) bool { + res := bytes.Compare(k1, k2) + return res != 0 && rng.Backwards == (res > 0) + } + for k, v := range s.mem { if isKeyOK(k) { memList = append(memList, KeyValue{ @@ -148,7 +159,7 @@ func (s *MemoryStore) seek(rng SeekRange, f func(k, v []byte)) { } } sort.Slice(memList, func(i, j int) bool { - return bytes.Compare(memList[i].Key, memList[j].Key) < 0 + return less(memList[i].Key, memList[j].Key) }) for _, kv := range memList { f(kv.Key, kv.Value) diff --git a/pkg/core/storage/store.go b/pkg/core/storage/store.go index 6b85903bf..65acfadbb 100644 --- a/pkg/core/storage/store.go +++ b/pkg/core/storage/store.go @@ -59,6 +59,10 @@ type SeekRange struct { // Empty Prefix and empty Start can be combined, which means seeking // through all keys in the DB. Start []byte + // Backwards denotes whether Seek direction should be reversed, i.e. + // whether seeking should be performed in a descending way. + // Backwards can be safely combined with Prefix and Start. + Backwards bool } // ErrKeyNotFound is an error returned by Store implementations diff --git a/pkg/core/storage/storeandbatch_test.go b/pkg/core/storage/storeandbatch_test.go index 9423db635..273408ea7 100644 --- a/pkg/core/storage/storeandbatch_test.go +++ b/pkg/core/storage/storeandbatch_test.go @@ -81,17 +81,27 @@ func testStoreSeek(t *testing.T, s Store) { require.NoError(t, s.Put(v.Key, v.Value)) } - check := func(t *testing.T, goodprefix, start []byte, goodkvs []KeyValue) { - // Seek result expected to be sorted in an ascending way. - sort.Slice(goodkvs, func(i, j int) bool { + check := func(t *testing.T, goodprefix, start []byte, goodkvs []KeyValue, backwards bool) { + // Seek result expected to be sorted in an ascending (for forwards seeking) or descending (for backwards seeking) way. + cmpFunc := func(i, j int) bool { return bytes.Compare(goodkvs[i].Key, goodkvs[j].Key) < 0 - }) + } + if backwards { + cmpFunc = func(i, j int) bool { + return bytes.Compare(goodkvs[i].Key, goodkvs[j].Key) > 0 + } + } + sort.Slice(goodkvs, cmpFunc) - actual := make([]KeyValue, 0, len(goodkvs)) - s.Seek(SeekRange{ + rng := SeekRange{ Prefix: goodprefix, Start: start, - }, func(k, v []byte) { + } + if backwards { + rng.Backwards = true + } + actual := make([]KeyValue, 0, len(goodkvs)) + s.Seek(rng, func(k, v []byte) { actual = append(actual, KeyValue{ Key: slice.Copy(k), Value: slice.Copy(v), @@ -101,59 +111,117 @@ func testStoreSeek(t *testing.T, s Store) { } t.Run("non-empty prefix, empty start", func(t *testing.T) { - t.Run("good", func(t *testing.T) { - // Given this prefix... - goodprefix := []byte("2") - // and empty start range... - start := []byte{} - // these pairs should be found. - goodkvs := []KeyValue{ - kvs[2], // key = "20" - kvs[3], // key = "21" - kvs[4], // key = "22" - } - check(t, goodprefix, start, goodkvs) + t.Run("forwards", func(t *testing.T) { + t.Run("good", func(t *testing.T) { + // Given this prefix... + goodprefix := []byte("2") + // and empty start range... + start := []byte{} + // these pairs should be found. + goodkvs := []KeyValue{ + kvs[2], // key = "20" + kvs[3], // key = "21" + kvs[4], // key = "22" + } + check(t, goodprefix, start, goodkvs, false) + }) + t.Run("no matching items", func(t *testing.T) { + goodprefix := []byte("0") + start := []byte{} + check(t, goodprefix, start, []KeyValue{}, false) + }) }) - t.Run("no matching items", func(t *testing.T) { - goodprefix := []byte("0") - start := []byte{} - check(t, goodprefix, start, []KeyValue{}) + + t.Run("backwards", func(t *testing.T) { + t.Run("good", func(t *testing.T) { + goodprefix := []byte("2") + start := []byte{} + goodkvs := []KeyValue{ + kvs[4], // key = "22" + kvs[3], // key = "21" + kvs[2], // key = "20" + } + check(t, goodprefix, start, goodkvs, true) + }) + t.Run("no matching items", func(t *testing.T) { + goodprefix := []byte("0") + start := []byte{} + check(t, goodprefix, start, []KeyValue{}, true) + }) }) }) t.Run("non-empty prefix, non-empty start", func(t *testing.T) { - t.Run("good", func(t *testing.T) { - goodprefix := []byte("2") - start := []byte("1") // start will be upended to goodprefix to start seek from - goodkvs := []KeyValue{ - kvs[3], // key = "21" - kvs[4], // key = "22" - } - check(t, goodprefix, start, goodkvs) + t.Run("forwards", func(t *testing.T) { + t.Run("good", func(t *testing.T) { + goodprefix := []byte("2") + start := []byte("1") // start will be upended to goodprefix to start seek from + goodkvs := []KeyValue{ + kvs[3], // key = "21" + kvs[4], // key = "22" + } + check(t, goodprefix, start, goodkvs, false) + }) + t.Run("no matching items", func(t *testing.T) { + goodprefix := []byte("2") + start := []byte("3") // start is more than all keys prefixed by '2'. + check(t, goodprefix, start, []KeyValue{}, false) + }) }) - t.Run("no matching items", func(t *testing.T) { - goodprefix := []byte("2") - start := []byte("3") // start will be upended to goodprefix to start seek from - check(t, goodprefix, start, []KeyValue{}) + t.Run("backwards", func(t *testing.T) { + t.Run("good", func(t *testing.T) { + goodprefix := []byte("2") + start := []byte("1") // start will be upended to goodprefix to start seek from + goodkvs := []KeyValue{ + kvs[3], // key = "21" + kvs[2], // key = "20" + } + check(t, goodprefix, start, goodkvs, true) + }) + t.Run("no matching items", func(t *testing.T) { + goodprefix := []byte("2") + start := []byte(".") // start is less than all keys prefixed by '2'. + check(t, goodprefix, start, []KeyValue{}, true) + }) }) }) t.Run("empty prefix, non-empty start", func(t *testing.T) { - t.Run("good", func(t *testing.T) { - goodprefix := []byte{} - start := []byte("21") - goodkvs := []KeyValue{ - kvs[3], // key = "21" - kvs[4], // key = "22" - kvs[5], // key = "30" - kvs[6], // key = "31" - } - check(t, goodprefix, start, goodkvs) + t.Run("forwards", func(t *testing.T) { + t.Run("good", func(t *testing.T) { + goodprefix := []byte{} + start := []byte("21") + goodkvs := []KeyValue{ + kvs[3], // key = "21" + kvs[4], // key = "22" + kvs[5], // key = "30" + kvs[6], // key = "31" + } + check(t, goodprefix, start, goodkvs, false) + }) + t.Run("no matching items", func(t *testing.T) { + goodprefix := []byte{} + start := []byte("32") // start is more than all keys. + check(t, goodprefix, start, []KeyValue{}, false) + }) }) - t.Run("no matching items", func(t *testing.T) { - goodprefix := []byte{} - start := []byte("32") - check(t, goodprefix, start, []KeyValue{}) + t.Run("backwards", func(t *testing.T) { + t.Run("good", func(t *testing.T) { + goodprefix := []byte{} + start := []byte("21") + goodkvs := []KeyValue{ + kvs[3], // key = "21" + kvs[2], // key = "20" + kvs[1], // key = "11" + kvs[0], // key = "10" + } + check(t, goodprefix, start, goodkvs, true) + }) + t.Run("no matching items", func(t *testing.T) { + goodprefix := []byte{} + start := []byte("0") // start is less than all keys. + check(t, goodprefix, start, []KeyValue{}, true) + }) }) }) @@ -162,7 +230,12 @@ func testStoreSeek(t *testing.T, s Store) { start := []byte{} goodkvs := make([]KeyValue, len(kvs)) copy(goodkvs, kvs) - check(t, goodprefix, start, goodkvs) + t.Run("forwards", func(t *testing.T) { + check(t, goodprefix, start, goodkvs, false) + }) + t.Run("backwards", func(t *testing.T) { + check(t, goodprefix, start, goodkvs, true) + }) }) require.NoError(t, s.Close()) From 2c46b8186409a2c6a18b3777e7344d57a13504f2 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 28 Dec 2021 18:33:28 +0300 Subject: [PATCH 6/6] native: optimize gas per vote calculation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GAS per vote records are being returned from Seek in ascending way, and we need the last record before the specified index which is close to the current chain's height (most of the time). To optimize we can iterate backwards strting from the last record using SeekBackwards. name old time/op new time/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 27.7µs ± 8% 30.3µs ± 2% +9.36% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 32.1µs ± 6% 37.4µs ± 5% +16.41% (p=0.000 n=9+10) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 60.9µs ± 8% 68.1µs ± 9% +11.72% (p=0.001 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 384µs ± 9% 437µs ± 2% +13.68% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 227µs ± 5% 255µs ± 6% +12.37% (p=0.000 n=9+9) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 222µs ± 9% 332µs ±47% +49.64% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 245µs ± 8% 293µs ±13% +19.64% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 731µs ±14% 1071µs ±57% +46.58% (p=0.001 n=9+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 7.45ms ±48% 9.83ms ±44% +32.00% (p=0.019 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 8.54ms ±46% 10.18ms ±46% ~ (p=0.218 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 7.35ms ±43% 7.72ms ±56% ~ (p=0.579 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 3.52ms ±32% 3.91ms ±24% ~ (p=0.684 n=10+10) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 25.2µs ± 5% 23.8µs ± 4% -5.25% (p=0.016 n=10+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 29.5µs ± 8% 26.2µs ± 3% -11.13% (p=0.000 n=10+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 44.9µs ±11% 40.2µs ± 9% -10.50% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 100µs ± 9% 83µs ±15% -16.75% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 88.4µs ± 8% 65.8µs ±17% -25.52% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 88.9µs ± 7% 65.4µs ±20% -26.44% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 94.4µs ± 9% 63.1µs ±18% -33.15% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 354µs ±35% 443µs ±49% ~ (p=0.190 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 469µs ±20% 227µs ±21% -51.66% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 468µs ±17% 267µs ±32% -42.92% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 480µs ±14% 253µs ±20% -47.25% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 497µs ±15% 311µs ±37% -37.39% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 27.2µs ± 8% 28.3µs ± 6% ~ (p=0.165 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 29.3µs ± 4% 30.2µs ± 8% ~ (p=0.089 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 51.7µs ±10% 54.3µs ±16% ~ (p=0.218 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 107µs ± 8% 112µs ±15% ~ (p=0.190 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 103µs ± 3% 97µs ±27% ~ (p=0.633 n=8+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 98.5µs ±10% 89.2µs ± 9% -9.46% (p=0.004 n=10+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 100µs ±10% 95µs ±14% ~ (p=0.243 n=9+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 222µs ± 7% 135µs ±40% -39.16% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 587µs ± 3% 448µs ±29% -23.58% (p=0.000 n=8+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 569µs ± 9% 438µs ±24% -22.98% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 578µs ±17% 436µs ±19% -24.49% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 683µs ±10% 480µs ±29% -29.76% (p=0.000 n=9+9) name old alloc/op new alloc/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 5.43kB ± 0% 4.83kB ± 0% -11.06% (p=0.000 n=9+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 5.74kB ± 0% 5.00kB ± 0% -12.91% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 9.90kB ± 0% 8.49kB ± 0% -14.24% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 62.9kB ± 0% 55.6kB ± 0% -11.63% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 37.5kB ± 0% 29.5kB ± 1% -21.31% (p=0.000 n=9+10) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 37.8kB ± 1% 30.0kB ± 2% -20.71% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 40.6kB ± 1% 32.0kB ± 1% -21.20% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 105kB ±13% 81kB ± 9% -22.81% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 374kB ± 1% 282kB ±24% -24.58% (p=0.000 n=8+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 376kB ± 9% 280kB ±11% -25.55% (p=0.000 n=8+9) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 359kB ± 9% 289kB ±19% -19.60% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 443kB ± 1% 362kB ± 8% -18.30% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 5.71kB ± 1% 5.17kB ± 1% -9.40% (p=0.000 n=10+7) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 6.04kB ± 1% 5.40kB ± 1% -10.61% (p=0.000 n=10+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 9.59kB ± 4% 8.19kB ± 1% -14.60% (p=0.000 n=10+8) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 41.6kB ± 2% 33.7kB ± 8% -18.98% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 29.2kB ± 2% 20.2kB ± 7% -30.94% (p=0.000 n=9+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 29.6kB ± 4% 20.3kB ± 8% -31.58% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 32.4kB ± 4% 21.7kB ± 5% -33.25% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 98.8kB ±17% 109.8kB ±41% ~ (p=0.353 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 220kB ± 2% 129kB ± 3% -41.40% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 219kB ± 4% 135kB ± 7% -38.52% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 223kB ± 5% 132kB ± 6% -40.69% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 263kB ± 5% 155kB ± 8% -41.23% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 6.19kB ± 1% 5.94kB ± 1% -4.15% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 6.51kB ± 0% 6.10kB ± 2% -6.27% (p=0.000 n=8+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 10.1kB ± 2% 9.7kB ± 3% -4.45% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 35.4kB ± 1% 33.6kB ± 3% -5.13% (p=0.000 n=9+9) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 28.1kB ± 3% 22.6kB ±10% -19.58% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 28.0kB ± 3% 23.7kB ± 9% -15.54% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 29.5kB ± 3% 24.8kB ±10% -16.08% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 53.7kB ± 4% 44.6kB ± 5% -16.83% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 207kB ± 2% 148kB ± 3% -28.58% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 206kB ± 2% 148kB ± 4% -28.20% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 208kB ± 1% 149kB ± 3% -28.07% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 234kB ± 2% 171kB ± 5% -26.71% (p=0.000 n=10+10) name old allocs/op new allocs/op delta NEO_GetGASPerVote/MemPS_10RewardRecords_1RewardDistance-8 129 ± 1% 95 ± 1% -26.33% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_10RewardRecords_10RewardDistance-8 139 ± 1% 100 ± 1% -27.85% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_100RewardDistance-8 225 ± 1% 155 ± 1% -31.11% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_10RewardRecords_1000RewardDistance-8 1.22k ± 3% 0.86k ± 1% -29.14% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1RewardDistance-8 863 ± 2% 468 ± 4% -45.72% (p=0.000 n=9+9) NEO_GetGASPerVote/MemPS_100RewardRecords_10RewardDistance-8 872 ± 2% 490 ± 7% -43.89% (p=0.000 n=8+10) NEO_GetGASPerVote/MemPS_100RewardRecords_100RewardDistance-8 973 ± 1% 550 ± 5% -43.44% (p=0.000 n=9+10) NEO_GetGASPerVote/MemPS_100RewardRecords_1000RewardDistance-8 2.11k ± 1% 1.37k ± 2% -35.11% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1RewardDistance-8 9.00k ± 1% 5.05k ± 1% -43.94% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_10RewardDistance-8 9.04k ± 1% 5.06k ± 1% -43.97% (p=0.000 n=10+9) NEO_GetGASPerVote/MemPS_1000RewardRecords_100RewardDistance-8 9.15k ± 1% 5.10k ± 2% -44.24% (p=0.000 n=10+10) NEO_GetGASPerVote/MemPS_1000RewardRecords_1000RewardDistance-8 10.2k ± 0% 5.8k ± 1% -42.60% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_10RewardRecords_1RewardDistance-8 136 ± 1% 103 ± 0% -24.15% (p=0.000 n=10+7) NEO_GetGASPerVote/BoltPS_10RewardRecords_10RewardDistance-8 146 ± 0% 107 ± 0% -26.71% (p=0.000 n=9+6) NEO_GetGASPerVote/BoltPS_10RewardRecords_100RewardDistance-8 232 ± 1% 164 ± 0% -29.46% (p=0.000 n=10+7) NEO_GetGASPerVote/BoltPS_10RewardRecords_1000RewardDistance-8 1.21k ± 1% 0.82k ± 1% -31.99% (p=0.000 n=10+8) NEO_GetGASPerVote/BoltPS_100RewardRecords_1RewardDistance-8 876 ± 0% 474 ± 0% -45.85% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_10RewardDistance-8 888 ± 0% 481 ± 0% -45.82% (p=0.000 n=9+10) NEO_GetGASPerVote/BoltPS_100RewardRecords_100RewardDistance-8 990 ± 1% 550 ± 0% -44.49% (p=0.000 n=10+9) NEO_GetGASPerVote/BoltPS_100RewardRecords_1000RewardDistance-8 2.29k ± 8% 1.70k ±19% -25.86% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1RewardDistance-8 8.18k ± 1% 4.15k ± 2% -49.33% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_10RewardDistance-8 8.19k ± 1% 4.14k ± 0% -49.41% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_100RewardDistance-8 8.29k ± 1% 4.21k ± 2% -49.20% (p=0.000 n=10+10) NEO_GetGASPerVote/BoltPS_1000RewardRecords_1000RewardDistance-8 9.31k ± 1% 4.84k ± 1% -48.05% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1RewardDistance-8 144 ± 1% 112 ± 3% -22.09% (p=0.000 n=8+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_10RewardDistance-8 153 ± 2% 116 ± 3% -23.66% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_100RewardDistance-8 236 ± 1% 172 ± 4% -27.33% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_10RewardRecords_1000RewardDistance-8 1.16k ± 0% 0.81k ± 1% -30.00% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_1RewardDistance-8 859 ± 1% 481 ± 2% -44.07% (p=0.000 n=9+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_10RewardDistance-8 872 ± 1% 478 ± 2% -45.14% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_100RewardRecords_100RewardDistance-8 968 ± 1% 550 ± 2% -43.21% (p=0.000 n=10+8) NEO_GetGASPerVote/LevelPS_100RewardRecords_1000RewardDistance-8 1.89k ± 1% 1.19k ± 1% -37.21% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1RewardDistance-8 8.14k ± 0% 4.22k ± 0% -48.10% (p=0.000 n=10+9) NEO_GetGASPerVote/LevelPS_1000RewardRecords_10RewardDistance-8 8.17k ± 0% 4.21k ± 1% -48.43% (p=0.000 n=10+10) NEO_GetGASPerVote/LevelPS_1000RewardRecords_100RewardDistance-8 8.23k ± 0% 4.28k ± 0% -48.00% (p=0.000 n=10+8) NEO_GetGASPerVote/LevelPS_1000RewardRecords_1000RewardDistance-8 9.14k ± 0% 4.90k ± 1% -46.45% (p=0.000 n=10+10) --- pkg/core/native/native_neo.go | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/pkg/core/native/native_neo.go b/pkg/core/native/native_neo.go index b90091909..9d5b61514 100644 --- a/pkg/core/native/native_neo.go +++ b/pkg/core/native/native_neo.go @@ -352,7 +352,7 @@ func (n *NEO) PostPersist(ic *interop.Context) error { if g, ok := n.gasPerVoteCache[cs[i].Key]; ok { r = &g } else { - reward := n.getGASPerVote(ic.DAO, key[:34], ic.Block.Index+1) + reward := n.getGASPerVote(ic.DAO, key[:34], []uint32{ic.Block.Index + 1}) r = &reward[0] } tmp.Add(tmp, r) @@ -383,16 +383,27 @@ func (n *NEO) PostPersist(ic *interop.Context) error { return nil } -func (n *NEO) getGASPerVote(d dao.DAO, key []byte, index ...uint32) []big.Int { - var max = make([]uint32, len(index)) - var reward = make([]big.Int, len(index)) - d.Seek(n.ID, storage.SeekRange{Prefix: key}, func(k, v []byte) { - if len(k) == 4 { +func (n *NEO) getGASPerVote(d dao.DAO, key []byte, indexes []uint32) []big.Int { + sort.Slice(indexes, func(i, j int) bool { + return indexes[i] < indexes[j] + }) + start := make([]byte, 4) + binary.BigEndian.PutUint32(start, indexes[len(indexes)-1]) + + need := len(indexes) + var reward = make([]big.Int, need) + collected := 0 + d.Seek(n.ID, storage.SeekRange{ + Prefix: key, + Start: start, + Backwards: true, + }, func(k, v []byte) { + if collected < need && len(k) == 4 { num := binary.BigEndian.Uint32(k) - for i, ind := range index { - if max[i] < num && num <= ind { - max[i] = num + for i, ind := range indexes { + if reward[i].Sign() == 0 && num <= ind { reward[i] = *bigint.FromBytes(v) + collected++ } } } @@ -638,8 +649,8 @@ func (n *NEO) calculateBonus(d dao.DAO, vote *keys.PublicKey, value *big.Int, st } var key = makeVoterKey(vote.Bytes()) - var reward = n.getGASPerVote(d, key, start, end) - var tmp = new(big.Int).Sub(&reward[1], &reward[0]) + var reward = n.getGASPerVote(d, key, []uint32{start, end}) + var tmp = (&reward[1]).Sub(&reward[1], &reward[0]) tmp.Mul(tmp, value) tmp.Div(tmp, bigVoterRewardFactor) tmp.Add(tmp, r)