From 9b841b9b8fc724cbf32e49f3a28667235ecc2fb9 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 18 Jan 2022 18:28:24 +0300 Subject: [PATCH] core, rpc: use Seek to iterate over NEP* transfers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The results are controversial a bit. MemoryPS is a special storage and the new approach doesn't help it to iterate through NEP17 transfers. However, on long distances (if transfers are requested for more than 1000 blocks, which is ~4.1 hours of mainnet chain) both LevelDB and BoltDB perform good enough with the new approach. Taking into account the fact that default value for the query period of `getnep17transfers` RPC handler is a week (~40K mainnet blocks) we can sacrifice MemoryPS performance in favour of BoltDB and LevelDB optimisation. Close #2263. Benchmark results: name old time/op new time/op delta Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take100Blocks-8 783µs ±13% 1762µs ± 4% +125.12% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take1000Blocks-8 6.91ms ± 2% 9.00ms ± 2% +30.28% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take100Blocks-8 1.43ms ± 8% 1.79ms ± 4% +24.93% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take1000Blocks-8 7.78ms ± 3% 8.93ms ± 2% +14.78% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take100Blocks-8 7.69ms ± 3% 1.73ms ±10% -77.50% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take1000Blocks-8 14.1ms ± 2% 9.0ms ± 2% -36.53% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take100Blocks-8 768µs ± 3% 801µs ± 2% +4.24% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take1000Blocks-8 8.03ms ± 2% 8.05ms ± 8% ~ (p=0.436 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take100Blocks-8 1.70ms ±16% 0.85ms ± 7% -49.85% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take1000Blocks-8 10.8ms ± 2% 8.1ms ± 2% -25.21% (p=0.000 n=10+8) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take100Blocks-8 10.8ms ± 2% 0.9ms ± 7% -92.12% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take1000Blocks-8 18.2ms ±13% 8.2ms ± 6% -54.95% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take100Blocks-8 874µs ± 6% 823µs ± 2% -5.81% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take1000Blocks-8 9.35ms ± 2% 8.14ms ± 2% -12.91% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take100Blocks-8 1.85ms ± 4% 0.89ms ±10% -51.68% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take1000Blocks-8 10.6ms ± 4% 8.2ms ± 2% -22.44% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take100Blocks-8 10.7ms ± 2% 0.9ms ± 2% -91.90% (p=0.000 n=8+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take1000Blocks-8 21.8ms ±15% 8.2ms ± 1% -62.25% (p=0.000 n=10+10) name old alloc/op new alloc/op delta Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take100Blocks-8 650kB ± 0% 1010kB ± 7% +55.45% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take1000Blocks-8 6.39MB ± 0% 9.89MB ± 1% +54.74% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take100Blocks-8 1.28MB ± 0% 1.14MB ± 0% -11.25% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take1000Blocks-8 7.02MB ± 0% 10.01MB ± 0% +42.54% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take100Blocks-8 7.02MB ± 0% 1.08MB ± 0% -84.64% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take1000Blocks-8 12.8MB ± 0% 10.0MB ± 0% -22.04% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take100Blocks-8 823kB ± 3% 866kB ± 5% +5.19% (p=0.001 n=9+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take1000Blocks-8 9.92MB ± 0% 9.85MB ± 0% -0.71% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take100Blocks-8 1.84MB ± 4% 1.01MB ± 5% -44.75% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take1000Blocks-8 11.0MB ± 0% 10.0MB ± 1% -8.45% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take100Blocks-8 11.0MB ± 0% 1.0MB ± 1% -90.55% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take1000Blocks-8 20.0MB ± 0% 10.1MB ± 0% -49.69% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take100Blocks-8 913kB ± 5% 907kB ± 6% ~ (p=0.497 n=9+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take1000Blocks-8 10.0MB ± 1% 10.0MB ± 1% -0.63% (p=0.001 n=10+8) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take100Blocks-8 1.92MB ± 2% 1.04MB ± 0% -45.53% (p=0.000 n=9+8) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take1000Blocks-8 11.1MB ± 1% 10.1MB ± 0% -9.22% (p=0.000 n=10+7) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take100Blocks-8 11.1MB ± 1% 1.0MB ± 0% -90.61% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take1000Blocks-8 20.4MB ± 1% 10.1MB ± 0% -50.46% (p=0.000 n=10+10) name old allocs/op new allocs/op delta Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take100Blocks-8 11.1k ± 0% 11.7k ± 0% +5.35% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take1000Blocks-8 110k ± 0% 110k ± 0% +0.55% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take100Blocks-8 22.0k ± 0% 11.9k ± 0% -46.14% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take1000Blocks-8 120k ± 0% 110k ± 0% -8.44% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take100Blocks-8 120k ± 0% 12k ± 0% -90.36% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take1000Blocks-8 219k ± 0% 110k ± 0% -49.73% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take100Blocks-8 11.3k ± 0% 11.2k ± 0% -1.06% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take1000Blocks-8 112k ± 0% 110k ± 0% -2.36% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take100Blocks-8 22.5k ± 0% 11.3k ± 0% -49.65% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take1000Blocks-8 124k ± 0% 110k ± 0% -11.08% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take100Blocks-8 124k ± 0% 11k ± 0% -90.84% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take1000Blocks-8 225k ± 0% 110k ± 0% -51.07% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take100Blocks-8 11.4k ± 0% 11.3k ± 0% -1.36% (p=0.000 n=9+9) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take1000Blocks-8 114k ± 0% 111k ± 0% -2.33% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take100Blocks-8 22.7k ± 0% 11.5k ± 0% -49.56% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take1000Blocks-8 125k ± 0% 111k ± 0% -11.12% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take100Blocks-8 125k ± 0% 11k ± 0% -90.82% (p=0.000 n=8+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take1000Blocks-8 229k ± 1% 111k ± 0% -51.42% (p=0.000 n=10+10) --- internal/fakechain/fakechain.go | 4 +- pkg/core/bench_test.go | 4 +- pkg/core/blockchain.go | 78 +++++++++------------------ pkg/core/blockchainer/blockchainer.go | 4 +- pkg/core/dao/dao.go | 63 ++++++++++++++++++---- pkg/core/state/tokens.go | 8 +++ pkg/rpc/server/server.go | 6 +-- 7 files changed, 97 insertions(+), 70 deletions(-) diff --git a/internal/fakechain/fakechain.go b/internal/fakechain/fakechain.go index 3573d1286..584861d70 100644 --- a/internal/fakechain/fakechain.go +++ b/internal/fakechain/fakechain.go @@ -286,12 +286,12 @@ func (chain *FakeChain) GetTokenLastUpdated(acc util.Uint160) (map[int32]uint32, } // ForEachNEP17Transfer implements Blockchainer interface. -func (chain *FakeChain) ForEachNEP11Transfer(util.Uint160, func(*state.NEP11Transfer) (bool, error)) error { +func (chain *FakeChain) ForEachNEP11Transfer(util.Uint160, uint64, func(*state.NEP11Transfer) (bool, error)) error { panic("TODO") } // ForEachNEP17Transfer implements Blockchainer interface. -func (chain *FakeChain) ForEachNEP17Transfer(util.Uint160, func(*state.NEP17Transfer) (bool, error)) error { +func (chain *FakeChain) ForEachNEP17Transfer(util.Uint160, uint64, func(*state.NEP17Transfer) (bool, error)) error { panic("TODO") } diff --git a/pkg/core/bench_test.go b/pkg/core/bench_test.go index bad3467d5..8bbb6b772 100644 --- a/pkg/core/bench_test.go +++ b/pkg/core/bench_test.go @@ -93,7 +93,7 @@ func benchmarkForEachNEP17Transfer(t *testing.B, ps storage.Store, startFromBloc newestB, err := bc.GetBlock(bc.GetHeaderHash(int(bc.BlockHeight()) - startFromBlock + 1)) require.NoError(t, err) - _ = newestB.Timestamp + newestTimestamp := newestB.Timestamp oldestB, err := bc.GetBlock(bc.GetHeaderHash(int(newestB.Index) - nBlocksToTake)) require.NoError(t, err) oldestTimestamp := oldestB.Timestamp @@ -102,7 +102,7 @@ func benchmarkForEachNEP17Transfer(t *testing.B, ps storage.Store, startFromBloc t.ReportAllocs() t.StartTimer() for i := 0; i < t.N; i++ { - require.NoError(t, bc.ForEachNEP17Transfer(acc, func(t *state.NEP17Transfer) (bool, error) { + require.NoError(t, bc.ForEachNEP17Transfer(acc, newestTimestamp, func(t *state.NEP17Transfer) (bool, error) { if t.Timestamp < oldestTimestamp { // iterating from newest to oldest, already have reached the needed height return false, nil diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 5c33bbf48..ac772cab5 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -1023,14 +1023,14 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error return } if !trData.Info.NewNEP11Batch { - err = kvcache.PutTokenTransferLog(acc, trData.Info.NextNEP11Batch, true, &trData.Log11) + err = kvcache.PutTokenTransferLog(acc, trData.Info.NextNEP11NewestTimestamp, trData.Info.NextNEP11Batch, true, &trData.Log11) if err != nil { aerdone <- err return } } if !trData.Info.NewNEP17Batch { - err = kvcache.PutTokenTransferLog(acc, trData.Info.NextNEP17Batch, false, &trData.Log17) + err = kvcache.PutTokenTransferLog(acc, trData.Info.NextNEP17NewestTimestamp, trData.Info.NextNEP17Batch, false, &trData.Log17) if err != nil { aerdone <- err return @@ -1334,18 +1334,18 @@ func (bc *Blockchain) processTokenTransfer(cache dao.DAO, transCache map[util.Ui } if !from.Equals(util.Uint160{}) { _ = nep17xfer.Amount.Neg(amount) // We already have the Int. - if appendTokenTransfer(cache, transCache, from, transfer, id, b.Index, isNEP11) != nil { + if appendTokenTransfer(cache, transCache, from, transfer, id, b.Index, b.Timestamp, isNEP11) != nil { return } } if !to.Equals(util.Uint160{}) { - _ = nep17xfer.Amount.Set(amount) // We already have the Int. - _ = appendTokenTransfer(cache, transCache, to, transfer, id, b.Index, isNEP11) // Nothing useful we can do. + _ = nep17xfer.Amount.Set(amount) // We already have the Int. + _ = appendTokenTransfer(cache, transCache, to, transfer, id, b.Index, b.Timestamp, isNEP11) // Nothing useful we can do. } } func appendTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData, addr util.Uint160, transfer io.Serializable, - token int32, bIndex uint32, isNEP11 bool) error { + token int32, bIndex uint32, bTimestamp uint64, isNEP11 bool) error { transferData, ok := transCache[addr] if !ok { balances, err := cache.GetTokenTransferInfo(addr) @@ -1353,14 +1353,14 @@ func appendTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData return err } if !balances.NewNEP11Batch { - trLog, err := cache.GetTokenTransferLog(addr, balances.NextNEP11Batch, true) + trLog, err := cache.GetTokenTransferLog(addr, balances.NextNEP11NewestTimestamp, balances.NextNEP11Batch, true) if err != nil { return err } transferData.Log11 = *trLog } if !balances.NewNEP17Batch { - trLog, err := cache.GetTokenTransferLog(addr, balances.NextNEP17Batch, false) + trLog, err := cache.GetTokenTransferLog(addr, balances.NextNEP17NewestTimestamp, balances.NextNEP17Batch, false) if err != nil { return err } @@ -1369,18 +1369,21 @@ func appendTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData transferData.Info = *balances } var ( - log *state.TokenTransferLog - newBatch *bool - nextBatch *uint32 + log *state.TokenTransferLog + newBatch *bool + nextBatch *uint32 + currTimestamp *uint64 ) if !isNEP11 { log = &transferData.Log17 newBatch = &transferData.Info.NewNEP17Batch nextBatch = &transferData.Info.NextNEP17Batch + currTimestamp = &transferData.Info.NextNEP17NewestTimestamp } else { log = &transferData.Log11 newBatch = &transferData.Info.NewNEP11Batch nextBatch = &transferData.Info.NextNEP11Batch + currTimestamp = &transferData.Info.NextNEP11NewestTimestamp } err := log.Append(transfer) if err != nil { @@ -1389,11 +1392,12 @@ func appendTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData transferData.Info.LastUpdated[token] = bIndex *newBatch = log.Size() >= state.TokenTransferBatchSize if *newBatch { - err = cache.PutTokenTransferLog(addr, *nextBatch, isNEP11, log) + err = cache.PutTokenTransferLog(addr, *currTimestamp, *nextBatch, isNEP11, log) if err != nil { return err } *nextBatch++ + *currTimestamp = bTimestamp // Put makes a copy of it anyway. log.Raw = log.Raw[:0] } @@ -1401,48 +1405,18 @@ func appendTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData return nil } -// ForEachNEP17Transfer executes f for each NEP-17 transfer in log. -func (bc *Blockchain) ForEachNEP17Transfer(acc util.Uint160, f func(*state.NEP17Transfer) (bool, error)) error { - balances, err := bc.dao.GetTokenTransferInfo(acc) - if err != nil { - return nil - } - for i := int(balances.NextNEP17Batch); i >= 0; i-- { - lg, err := bc.dao.GetTokenTransferLog(acc, uint32(i), false) - if err != nil { - return nil - } - cont, err := lg.ForEachNEP17(f) - if err != nil { - return err - } - if !cont { - break - } - } - return nil +// ForEachNEP17Transfer executes f for each NEP-17 transfer in log starting from +// the transfer with the newest timestamp up to the oldest transfer. It continues +// iteration until false is returned from f. The last non-nil error is returned. +func (bc *Blockchain) ForEachNEP17Transfer(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP17Transfer) (bool, error)) error { + return bc.dao.SeekNEP17TransferLog(acc, newestTimestamp, f) } -// ForEachNEP11Transfer executes f for each NEP-11 transfer in log. -func (bc *Blockchain) ForEachNEP11Transfer(acc util.Uint160, f func(*state.NEP11Transfer) (bool, error)) error { - balances, err := bc.dao.GetTokenTransferInfo(acc) - if err != nil { - return nil - } - for i := int(balances.NextNEP11Batch); i >= 0; i-- { - lg, err := bc.dao.GetTokenTransferLog(acc, uint32(i), true) - if err != nil { - return nil - } - cont, err := lg.ForEachNEP11(f) - if err != nil { - return err - } - if !cont { - break - } - } - return nil +// ForEachNEP11Transfer executes f for each NEP-11 transfer in log starting from +// the transfer with the newest timestamp up to the oldest transfer. It continues +// iteration until false is returned from f. The last non-nil error is returned. +func (bc *Blockchain) ForEachNEP11Transfer(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP11Transfer) (bool, error)) error { + return bc.dao.SeekNEP11TransferLog(acc, newestTimestamp, f) } // GetNEP17Contracts returns the list of deployed NEP-17 contracts. diff --git a/pkg/core/blockchainer/blockchainer.go b/pkg/core/blockchainer/blockchainer.go index 710d5dfb1..d2320730a 100644 --- a/pkg/core/blockchainer/blockchainer.go +++ b/pkg/core/blockchainer/blockchainer.go @@ -36,8 +36,8 @@ type Blockchainer interface { GetContractScriptHash(id int32) (util.Uint160, error) GetEnrollments() ([]state.Validator, error) GetGoverningTokenBalance(acc util.Uint160) (*big.Int, uint32) - ForEachNEP11Transfer(util.Uint160, func(*state.NEP11Transfer) (bool, error)) error - ForEachNEP17Transfer(util.Uint160, func(*state.NEP17Transfer) (bool, error)) error + ForEachNEP11Transfer(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP11Transfer) (bool, error)) error + ForEachNEP17Transfer(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP17Transfer) (bool, error)) error GetHeaderHash(int) util.Uint256 GetHeader(hash util.Uint256) (*block.Header, error) CurrentHeaderHash() util.Uint256 diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index 9e4138264..aff495107 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -42,7 +42,7 @@ type DAO interface { GetCurrentHeaderHeight() (i uint32, h util.Uint256, err error) GetHeaderHashes() ([]util.Uint256, error) GetTokenTransferInfo(acc util.Uint160) (*state.TokenTransferInfo, error) - GetTokenTransferLog(acc util.Uint160, index uint32, isNEP11 bool) (*state.TokenTransferLog, error) + GetTokenTransferLog(acc util.Uint160, start uint64, index uint32, isNEP11 bool) (*state.TokenTransferLog, error) GetStateSyncPoint() (uint32, error) GetStateSyncCurrentBlockHeight() (uint32, error) GetStorageItem(id int32, key []byte) state.StorageItem @@ -57,7 +57,7 @@ type DAO interface { PutContractID(id int32, hash util.Uint160) error PutCurrentHeader(hashAndIndex []byte) error PutTokenTransferInfo(acc util.Uint160, bs *state.TokenTransferInfo) error - PutTokenTransferLog(acc util.Uint160, index uint32, isNEP11 bool, lg *state.TokenTransferLog) error + PutTokenTransferLog(acc util.Uint160, start uint64, index uint32, isNEP11 bool, lg *state.TokenTransferLog) error PutStateSyncPoint(p uint32) error PutStateSyncCurrentBlockHeight(h uint32) error PutStorageItem(id int32, key []byte, si state.StorageItem) error @@ -180,21 +180,66 @@ func (dao *Simple) putTokenTransferInfo(acc util.Uint160, bs *state.TokenTransfe // -- start transfer log. -func getTokenTransferLogKey(acc util.Uint160, index uint32, isNEP11 bool) []byte { - key := make([]byte, 1+util.Uint160Size+4) +func getTokenTransferLogKey(acc util.Uint160, newestTimestamp uint64, index uint32, isNEP11 bool) []byte { + key := make([]byte, 1+util.Uint160Size+8+4) if isNEP11 { key[0] = byte(storage.STNEP11Transfers) } else { key[0] = byte(storage.STNEP17Transfers) } copy(key[1:], acc.BytesBE()) - binary.LittleEndian.PutUint32(key[util.Uint160Size:], index) + binary.BigEndian.PutUint64(key[1+util.Uint160Size:], newestTimestamp) + binary.BigEndian.PutUint32(key[1+util.Uint160Size+8:], index) return key } +// SeekNEP17TransferLog executes f for each NEP-17 transfer in log starting from +// the transfer with the newest timestamp up to the oldest transfer. It continues +// iteration until false is returned from f. The last non-nil error is returned. +func (dao *Simple) SeekNEP17TransferLog(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP17Transfer) (bool, error)) error { + key := getTokenTransferLogKey(acc, newestTimestamp, 0, false) + prefixLen := 1 + util.Uint160Size + var seekErr error + dao.Store.Seek(storage.SeekRange{ + Prefix: key[:prefixLen], + Start: key[prefixLen : prefixLen+8], + Backwards: true, + }, func(k, v []byte) bool { + lg := &state.TokenTransferLog{Raw: v} + cont, err := lg.ForEachNEP17(f) + if err != nil { + seekErr = err + } + return cont + }) + return seekErr +} + +// SeekNEP11TransferLog executes f for each NEP-11 transfer in log starting from +// the transfer with the newest timestamp up to the oldest transfer. It continues +// iteration until false is returned from f. The last non-nil error is returned. +func (dao *Simple) SeekNEP11TransferLog(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP11Transfer) (bool, error)) error { + key := getTokenTransferLogKey(acc, newestTimestamp, 0, true) + prefixLen := 1 + util.Uint160Size + var seekErr error + dao.Store.Seek(storage.SeekRange{ + Prefix: key[:prefixLen], + Start: key[prefixLen : prefixLen+8], + Backwards: true, + }, func(k, v []byte) bool { + lg := &state.TokenTransferLog{Raw: v} + cont, err := lg.ForEachNEP11(f) + if err != nil { + seekErr = err + } + return cont + }) + return seekErr +} + // GetTokenTransferLog retrieves transfer log from the cache. -func (dao *Simple) GetTokenTransferLog(acc util.Uint160, index uint32, isNEP11 bool) (*state.TokenTransferLog, error) { - key := getTokenTransferLogKey(acc, index, isNEP11) +func (dao *Simple) GetTokenTransferLog(acc util.Uint160, newestTimestamp uint64, index uint32, isNEP11 bool) (*state.TokenTransferLog, error) { + key := getTokenTransferLogKey(acc, newestTimestamp, index, isNEP11) value, err := dao.Store.Get(key) if err != nil { if err == storage.ErrKeyNotFound { @@ -206,8 +251,8 @@ func (dao *Simple) GetTokenTransferLog(acc util.Uint160, index uint32, isNEP11 b } // PutTokenTransferLog saves given transfer log in the cache. -func (dao *Simple) PutTokenTransferLog(acc util.Uint160, index uint32, isNEP11 bool, lg *state.TokenTransferLog) error { - key := getTokenTransferLogKey(acc, index, isNEP11) +func (dao *Simple) PutTokenTransferLog(acc util.Uint160, start uint64, index uint32, isNEP11 bool, lg *state.TokenTransferLog) error { + key := getTokenTransferLogKey(acc, start, index, isNEP11) return dao.Store.Put(key, lg.Raw) } diff --git a/pkg/core/state/tokens.go b/pkg/core/state/tokens.go index 2d1dc8f51..9470fa845 100644 --- a/pkg/core/state/tokens.go +++ b/pkg/core/state/tokens.go @@ -53,6 +53,10 @@ type TokenTransferInfo struct { NextNEP11Batch uint32 // NextNEP17Batch stores the index of the next NEP-17 transfer batch. NextNEP17Batch uint32 + // NextNEP11NewestTimestamp stores the block timestamp of the first NEP-11 transfer in raw. + NextNEP11NewestTimestamp uint64 + // NextNEP17NewestTimestamp stores the block timestamp of the first NEP-17 transfer in raw. + NextNEP17NewestTimestamp uint64 // NewNEP11Batch is true if batch with the `NextNEP11Batch` index should be created. NewNEP11Batch bool // NewNEP17Batch is true if batch with the `NextNEP17Batch` index should be created. @@ -72,6 +76,8 @@ func NewTokenTransferInfo() *TokenTransferInfo { func (bs *TokenTransferInfo) DecodeBinary(r *io.BinReader) { bs.NextNEP11Batch = r.ReadU32LE() bs.NextNEP17Batch = r.ReadU32LE() + bs.NextNEP11NewestTimestamp = r.ReadU64LE() + bs.NextNEP17NewestTimestamp = r.ReadU64LE() bs.NewNEP11Batch = r.ReadBool() bs.NewNEP17Batch = r.ReadBool() lenBalances := r.ReadVarUint() @@ -87,6 +93,8 @@ func (bs *TokenTransferInfo) DecodeBinary(r *io.BinReader) { func (bs *TokenTransferInfo) EncodeBinary(w *io.BinWriter) { w.WriteU32LE(bs.NextNEP11Batch) w.WriteU32LE(bs.NextNEP17Batch) + w.WriteU64LE(bs.NextNEP11NewestTimestamp) + w.WriteU64LE(bs.NextNEP17NewestTimestamp) w.WriteBool(bs.NewNEP11Batch) w.WriteBool(bs.NewNEP17Batch) w.WriteVarUint(uint64(len(bs.LastUpdated))) diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index cd5f25763..a8e11deb9 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -1019,7 +1019,7 @@ func (s *Server) getTokenTransfers(ps request.Params, isNEP11 bool) (interface{} return received, sent, !(limit != 0 && resCount >= limit), nil } if !isNEP11 { - err = s.chain.ForEachNEP17Transfer(u, func(tr *state.NEP17Transfer) (bool, error) { + err = s.chain.ForEachNEP17Transfer(u, end, func(tr *state.NEP17Transfer) (bool, error) { r, s, res, err := handleTransfer(tr) if err == nil { if r != nil { @@ -1032,7 +1032,7 @@ func (s *Server) getTokenTransfers(ps request.Params, isNEP11 bool) (interface{} return res, err }) } else { - err = s.chain.ForEachNEP11Transfer(u, func(tr *state.NEP11Transfer) (bool, error) { + err = s.chain.ForEachNEP11Transfer(u, end, func(tr *state.NEP11Transfer) (bool, error) { r, s, res, err := handleTransfer(&tr.NEP17Transfer) if err == nil { id := hex.EncodeToString(tr.ID) @@ -1047,7 +1047,7 @@ func (s *Server) getTokenTransfers(ps request.Params, isNEP11 bool) (interface{} }) } if err != nil { - return nil, response.NewInternalServerError("invalid transfer log", err) + return nil, response.NewInternalServerError(fmt.Sprintf("invalid transfer log: %v", err), err) } return bs, nil }