core, rpc: use Seek to iterate over NEP* transfers

The results are controversial a bit. MemoryPS is a special storage
and the new approach doesn't help it to iterate through NEP17 transfers.
However, on long distances (if transfers are requested for more than
1000 blocks, which is ~4.1 hours of mainnet chain) both LevelDB and
BoltDB perform good enough with the new approach. Taking into account
the fact that default value for the query period of `getnep17transfers`
RPC handler is a week (~40K mainnet blocks) we can sacrifice MemoryPS
performance in favour of BoltDB and LevelDB optimisation.

Close #2263.

Benchmark results:

name                                                                           old time/op    new time/op    delta
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take100Blocks-8           783µs ±13%    1762µs ± 4%  +125.12%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take1000Blocks-8         6.91ms ± 2%    9.00ms ± 2%   +30.28%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take100Blocks-8        1.43ms ± 8%    1.79ms ± 4%   +24.93%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take1000Blocks-8       7.78ms ± 3%    8.93ms ± 2%   +14.78%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take100Blocks-8       7.69ms ± 3%    1.73ms ±10%   -77.50%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take1000Blocks-8      14.1ms ± 2%     9.0ms ± 2%   -36.53%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take100Blocks-8          768µs ± 3%     801µs ± 2%    +4.24%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take1000Blocks-8        8.03ms ± 2%    8.05ms ± 8%      ~     (p=0.436 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take100Blocks-8       1.70ms ±16%    0.85ms ± 7%   -49.85%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take1000Blocks-8      10.8ms ± 2%     8.1ms ± 2%   -25.21%  (p=0.000 n=10+8)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take100Blocks-8      10.8ms ± 2%     0.9ms ± 7%   -92.12%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take1000Blocks-8     18.2ms ±13%     8.2ms ± 6%   -54.95%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take100Blocks-8         874µs ± 6%     823µs ± 2%    -5.81%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take1000Blocks-8       9.35ms ± 2%    8.14ms ± 2%   -12.91%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take100Blocks-8      1.85ms ± 4%    0.89ms ±10%   -51.68%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take1000Blocks-8     10.6ms ± 4%     8.2ms ± 2%   -22.44%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take100Blocks-8     10.7ms ± 2%     0.9ms ± 2%   -91.90%  (p=0.000 n=8+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take1000Blocks-8    21.8ms ±15%     8.2ms ± 1%   -62.25%  (p=0.000 n=10+10)

name                                                                           old alloc/op   new alloc/op   delta
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take100Blocks-8           650kB ± 0%    1010kB ± 7%   +55.45%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take1000Blocks-8         6.39MB ± 0%    9.89MB ± 1%   +54.74%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take100Blocks-8        1.28MB ± 0%    1.14MB ± 0%   -11.25%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take1000Blocks-8       7.02MB ± 0%   10.01MB ± 0%   +42.54%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take100Blocks-8       7.02MB ± 0%    1.08MB ± 0%   -84.64%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take1000Blocks-8      12.8MB ± 0%    10.0MB ± 0%   -22.04%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take100Blocks-8          823kB ± 3%     866kB ± 5%    +5.19%  (p=0.001 n=9+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take1000Blocks-8        9.92MB ± 0%    9.85MB ± 0%    -0.71%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take100Blocks-8       1.84MB ± 4%    1.01MB ± 5%   -44.75%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take1000Blocks-8      11.0MB ± 0%    10.0MB ± 1%    -8.45%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take100Blocks-8      11.0MB ± 0%     1.0MB ± 1%   -90.55%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take1000Blocks-8     20.0MB ± 0%    10.1MB ± 0%   -49.69%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take100Blocks-8         913kB ± 5%     907kB ± 6%      ~     (p=0.497 n=9+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take1000Blocks-8       10.0MB ± 1%    10.0MB ± 1%    -0.63%  (p=0.001 n=10+8)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take100Blocks-8      1.92MB ± 2%    1.04MB ± 0%   -45.53%  (p=0.000 n=9+8)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take1000Blocks-8     11.1MB ± 1%    10.1MB ± 0%    -9.22%  (p=0.000 n=10+7)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take100Blocks-8     11.1MB ± 1%     1.0MB ± 0%   -90.61%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take1000Blocks-8    20.4MB ± 1%    10.1MB ± 0%   -50.46%  (p=0.000 n=10+10)

name                                                                           old allocs/op  new allocs/op  delta
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take100Blocks-8           11.1k ± 0%     11.7k ± 0%    +5.35%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take1000Blocks-8           110k ± 0%      110k ± 0%    +0.55%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take100Blocks-8         22.0k ± 0%     11.9k ± 0%   -46.14%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take1000Blocks-8         120k ± 0%      110k ± 0%    -8.44%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take100Blocks-8         120k ± 0%       12k ± 0%   -90.36%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take1000Blocks-8        219k ± 0%      110k ± 0%   -49.73%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take100Blocks-8          11.3k ± 0%     11.2k ± 0%    -1.06%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take1000Blocks-8          112k ± 0%      110k ± 0%    -2.36%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take100Blocks-8        22.5k ± 0%     11.3k ± 0%   -49.65%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take1000Blocks-8        124k ± 0%      110k ± 0%   -11.08%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take100Blocks-8        124k ± 0%       11k ± 0%   -90.84%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take1000Blocks-8       225k ± 0%      110k ± 0%   -51.07%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take100Blocks-8         11.4k ± 0%     11.3k ± 0%    -1.36%  (p=0.000 n=9+9)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take1000Blocks-8         114k ± 0%      111k ± 0%    -2.33%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take100Blocks-8       22.7k ± 0%     11.5k ± 0%   -49.56%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take1000Blocks-8       125k ± 0%      111k ± 0%   -11.12%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take100Blocks-8       125k ± 0%       11k ± 0%   -90.82%  (p=0.000 n=8+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take1000Blocks-8      229k ± 1%      111k ± 0%   -51.42%  (p=0.000 n=10+10)

internal/fakechain/fakechain.go

@@ -286,12 +286,12 @@ func (chain *FakeChain) GetTokenLastUpdated(acc util.Uint160) (map[int32]uint32,
 }
 
 // ForEachNEP17Transfer implements Blockchainer interface.
-func (chain *FakeChain) ForEachNEP11Transfer(util.Uint160, func(*state.NEP11Transfer) (bool, error)) error {
+func (chain *FakeChain) ForEachNEP11Transfer(util.Uint160, uint64, func(*state.NEP11Transfer) (bool, error)) error {
 	panic("TODO")
 }
 
 // ForEachNEP17Transfer implements Blockchainer interface.
-func (chain *FakeChain) ForEachNEP17Transfer(util.Uint160, func(*state.NEP17Transfer) (bool, error)) error {
+func (chain *FakeChain) ForEachNEP17Transfer(util.Uint160, uint64, func(*state.NEP17Transfer) (bool, error)) error {
 	panic("TODO")
 }
 

pkg/core/bench_test.go

@@ -93,7 +93,7 @@ func benchmarkForEachNEP17Transfer(t *testing.B, ps storage.Store, startFromBloc
 
 	newestB, err := bc.GetBlock(bc.GetHeaderHash(int(bc.BlockHeight()) - startFromBlock + 1))
 	require.NoError(t, err)
-	_ = newestB.Timestamp
+	newestTimestamp := newestB.Timestamp
 	oldestB, err := bc.GetBlock(bc.GetHeaderHash(int(newestB.Index) - nBlocksToTake))
 	require.NoError(t, err)
 	oldestTimestamp := oldestB.Timestamp
@@ -102,7 +102,7 @@ func benchmarkForEachNEP17Transfer(t *testing.B, ps storage.Store, startFromBloc
 	t.ReportAllocs()
 	t.StartTimer()
 	for i := 0; i < t.N; i++ {
-		require.NoError(t, bc.ForEachNEP17Transfer(acc, func(t *state.NEP17Transfer) (bool, error) {
+		require.NoError(t, bc.ForEachNEP17Transfer(acc, newestTimestamp, func(t *state.NEP17Transfer) (bool, error) {
 			if t.Timestamp < oldestTimestamp {
 				// iterating from newest to oldest, already have reached the needed height
 				return false, nil

pkg/core/blockchain.go

@@ -1023,14 +1023,14 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error
 				return
 			}
 			if !trData.Info.NewNEP11Batch {
-				err = kvcache.PutTokenTransferLog(acc, trData.Info.NextNEP11Batch, true, &trData.Log11)
+				err = kvcache.PutTokenTransferLog(acc, trData.Info.NextNEP11NewestTimestamp, trData.Info.NextNEP11Batch, true, &trData.Log11)
 				if err != nil {
 					aerdone <- err
 					return
 				}
 			}
 			if !trData.Info.NewNEP17Batch {
-				err = kvcache.PutTokenTransferLog(acc, trData.Info.NextNEP17Batch, false, &trData.Log17)
+				err = kvcache.PutTokenTransferLog(acc, trData.Info.NextNEP17NewestTimestamp, trData.Info.NextNEP17Batch, false, &trData.Log17)
 				if err != nil {
 					aerdone <- err
 					return
@@ -1334,18 +1334,18 @@ func (bc *Blockchain) processTokenTransfer(cache dao.DAO, transCache map[util.Ui
 	}
 	if !from.Equals(util.Uint160{}) {
 		_ = nep17xfer.Amount.Neg(amount) // We already have the Int.
-		if appendTokenTransfer(cache, transCache, from, transfer, id, b.Index, isNEP11) != nil {
+		if appendTokenTransfer(cache, transCache, from, transfer, id, b.Index, b.Timestamp, isNEP11) != nil {
 			return
 		}
 	}
 	if !to.Equals(util.Uint160{}) {
 		_ = nep17xfer.Amount.Set(amount)                                                            // We already have the Int.
-		_ = appendTokenTransfer(cache, transCache, to, transfer, id, b.Index, isNEP11) // Nothing useful we can do.
+		_ = appendTokenTransfer(cache, transCache, to, transfer, id, b.Index, b.Timestamp, isNEP11) // Nothing useful we can do.
 	}
 }
 
 func appendTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData, addr util.Uint160, transfer io.Serializable,
-	token int32, bIndex uint32, isNEP11 bool) error {
+	token int32, bIndex uint32, bTimestamp uint64, isNEP11 bool) error {
 	transferData, ok := transCache[addr]
 	if !ok {
 		balances, err := cache.GetTokenTransferInfo(addr)
@@ -1353,14 +1353,14 @@ func appendTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData
 			return err
 		}
 		if !balances.NewNEP11Batch {
-			trLog, err := cache.GetTokenTransferLog(addr, balances.NextNEP11Batch, true)
+			trLog, err := cache.GetTokenTransferLog(addr, balances.NextNEP11NewestTimestamp, balances.NextNEP11Batch, true)
 			if err != nil {
 				return err
 			}
 			transferData.Log11 = *trLog
 		}
 		if !balances.NewNEP17Batch {
-			trLog, err := cache.GetTokenTransferLog(addr, balances.NextNEP17Batch, false)
+			trLog, err := cache.GetTokenTransferLog(addr, balances.NextNEP17NewestTimestamp, balances.NextNEP17Batch, false)
 			if err != nil {
 				return err
 			}
@@ -1372,15 +1372,18 @@ func appendTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData
 		log           *state.TokenTransferLog
 		newBatch      *bool
 		nextBatch     *uint32
+		currTimestamp *uint64
 	)
 	if !isNEP11 {
 		log = &transferData.Log17
 		newBatch = &transferData.Info.NewNEP17Batch
 		nextBatch = &transferData.Info.NextNEP17Batch
+		currTimestamp = &transferData.Info.NextNEP17NewestTimestamp
 	} else {
 		log = &transferData.Log11
 		newBatch = &transferData.Info.NewNEP11Batch
 		nextBatch = &transferData.Info.NextNEP11Batch
+		currTimestamp = &transferData.Info.NextNEP11NewestTimestamp
 	}
 	err := log.Append(transfer)
 	if err != nil {
@@ -1389,11 +1392,12 @@ func appendTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData
 	transferData.Info.LastUpdated[token] = bIndex
 	*newBatch = log.Size() >= state.TokenTransferBatchSize
 	if *newBatch {
-		err = cache.PutTokenTransferLog(addr, *nextBatch, isNEP11, log)
+		err = cache.PutTokenTransferLog(addr, *currTimestamp, *nextBatch, isNEP11, log)
 		if err != nil {
 			return err
 		}
 		*nextBatch++
+		*currTimestamp = bTimestamp
 		// Put makes a copy of it anyway.
 		log.Raw = log.Raw[:0]
 	}
@@ -1401,48 +1405,18 @@ func appendTokenTransfer(cache dao.DAO, transCache map[util.Uint160]transferData
 	return nil
 }
 
-// ForEachNEP17Transfer executes f for each NEP-17 transfer in log.
+// ForEachNEP17Transfer executes f for each NEP-17 transfer in log starting from
-func (bc *Blockchain) ForEachNEP17Transfer(acc util.Uint160, f func(*state.NEP17Transfer) (bool, error)) error {
+// the transfer with the newest timestamp up to the oldest transfer. It continues
-	balances, err := bc.dao.GetTokenTransferInfo(acc)
+// iteration until false is returned from f. The last non-nil error is returned.
-	if err != nil {
+func (bc *Blockchain) ForEachNEP17Transfer(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP17Transfer) (bool, error)) error {
-		return nil
+	return bc.dao.SeekNEP17TransferLog(acc, newestTimestamp, f)
-	}
-	for i := int(balances.NextNEP17Batch); i >= 0; i-- {
-		lg, err := bc.dao.GetTokenTransferLog(acc, uint32(i), false)
-		if err != nil {
-			return nil
-		}
-		cont, err := lg.ForEachNEP17(f)
-		if err != nil {
-			return err
-		}
-		if !cont {
-			break
-		}
-	}
-	return nil
 }
 
-// ForEachNEP11Transfer executes f for each NEP-11 transfer in log.
+// ForEachNEP11Transfer executes f for each NEP-11 transfer in log starting from
-func (bc *Blockchain) ForEachNEP11Transfer(acc util.Uint160, f func(*state.NEP11Transfer) (bool, error)) error {
+// the transfer with the newest timestamp up to the oldest transfer. It continues
-	balances, err := bc.dao.GetTokenTransferInfo(acc)
+// iteration until false is returned from f. The last non-nil error is returned.
-	if err != nil {
+func (bc *Blockchain) ForEachNEP11Transfer(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP11Transfer) (bool, error)) error {
-		return nil
+	return bc.dao.SeekNEP11TransferLog(acc, newestTimestamp, f)
-	}
-	for i := int(balances.NextNEP11Batch); i >= 0; i-- {
-		lg, err := bc.dao.GetTokenTransferLog(acc, uint32(i), true)
-		if err != nil {
-			return nil
-		}
-		cont, err := lg.ForEachNEP11(f)
-		if err != nil {
-			return err
-		}
-		if !cont {
-			break
-		}
-	}
-	return nil
 }
 
 // GetNEP17Contracts returns the list of deployed NEP-17 contracts.

pkg/core/blockchainer/blockchainer.go

@@ -36,8 +36,8 @@ type Blockchainer interface {
 	GetContractScriptHash(id int32) (util.Uint160, error)
 	GetEnrollments() ([]state.Validator, error)
 	GetGoverningTokenBalance(acc util.Uint160) (*big.Int, uint32)
-	ForEachNEP11Transfer(util.Uint160, func(*state.NEP11Transfer) (bool, error)) error
+	ForEachNEP11Transfer(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP11Transfer) (bool, error)) error
-	ForEachNEP17Transfer(util.Uint160, func(*state.NEP17Transfer) (bool, error)) error
+	ForEachNEP17Transfer(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP17Transfer) (bool, error)) error
 	GetHeaderHash(int) util.Uint256
 	GetHeader(hash util.Uint256) (*block.Header, error)
 	CurrentHeaderHash() util.Uint256

pkg/core/dao/dao.go

@@ -42,7 +42,7 @@ type DAO interface {
 	GetCurrentHeaderHeight() (i uint32, h util.Uint256, err error)
 	GetHeaderHashes() ([]util.Uint256, error)
 	GetTokenTransferInfo(acc util.Uint160) (*state.TokenTransferInfo, error)
-	GetTokenTransferLog(acc util.Uint160, index uint32, isNEP11 bool) (*state.TokenTransferLog, error)
+	GetTokenTransferLog(acc util.Uint160, start uint64, index uint32, isNEP11 bool) (*state.TokenTransferLog, error)
 	GetStateSyncPoint() (uint32, error)
 	GetStateSyncCurrentBlockHeight() (uint32, error)
 	GetStorageItem(id int32, key []byte) state.StorageItem
@@ -57,7 +57,7 @@ type DAO interface {
 	PutContractID(id int32, hash util.Uint160) error
 	PutCurrentHeader(hashAndIndex []byte) error
 	PutTokenTransferInfo(acc util.Uint160, bs *state.TokenTransferInfo) error
-	PutTokenTransferLog(acc util.Uint160, index uint32, isNEP11 bool, lg *state.TokenTransferLog) error
+	PutTokenTransferLog(acc util.Uint160, start uint64, index uint32, isNEP11 bool, lg *state.TokenTransferLog) error
 	PutStateSyncPoint(p uint32) error
 	PutStateSyncCurrentBlockHeight(h uint32) error
 	PutStorageItem(id int32, key []byte, si state.StorageItem) error
@@ -180,21 +180,66 @@ func (dao *Simple) putTokenTransferInfo(acc util.Uint160, bs *state.TokenTransfe
 
 // -- start transfer log.
 
-func getTokenTransferLogKey(acc util.Uint160, index uint32, isNEP11 bool) []byte {
+func getTokenTransferLogKey(acc util.Uint160, newestTimestamp uint64, index uint32, isNEP11 bool) []byte {
-	key := make([]byte, 1+util.Uint160Size+4)
+	key := make([]byte, 1+util.Uint160Size+8+4)
 	if isNEP11 {
 		key[0] = byte(storage.STNEP11Transfers)
 	} else {
 		key[0] = byte(storage.STNEP17Transfers)
 	}
 	copy(key[1:], acc.BytesBE())
-	binary.LittleEndian.PutUint32(key[util.Uint160Size:], index)
+	binary.BigEndian.PutUint64(key[1+util.Uint160Size:], newestTimestamp)
+	binary.BigEndian.PutUint32(key[1+util.Uint160Size+8:], index)
 	return key
 }
 
+// SeekNEP17TransferLog executes f for each NEP-17 transfer in log starting from
+// the transfer with the newest timestamp up to the oldest transfer. It continues
+// iteration until false is returned from f. The last non-nil error is returned.
+func (dao *Simple) SeekNEP17TransferLog(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP17Transfer) (bool, error)) error {
+	key := getTokenTransferLogKey(acc, newestTimestamp, 0, false)
+	prefixLen := 1 + util.Uint160Size
+	var seekErr error
+	dao.Store.Seek(storage.SeekRange{
+		Prefix:    key[:prefixLen],
+		Start:     key[prefixLen : prefixLen+8],
+		Backwards: true,
+	}, func(k, v []byte) bool {
+		lg := &state.TokenTransferLog{Raw: v}
+		cont, err := lg.ForEachNEP17(f)
+		if err != nil {
+			seekErr = err
+		}
+		return cont
+	})
+	return seekErr
+}
+
+// SeekNEP11TransferLog executes f for each NEP-11 transfer in log starting from
+// the transfer with the newest timestamp up to the oldest transfer. It continues
+// iteration until false is returned from f. The last non-nil error is returned.
+func (dao *Simple) SeekNEP11TransferLog(acc util.Uint160, newestTimestamp uint64, f func(*state.NEP11Transfer) (bool, error)) error {
+	key := getTokenTransferLogKey(acc, newestTimestamp, 0, true)
+	prefixLen := 1 + util.Uint160Size
+	var seekErr error
+	dao.Store.Seek(storage.SeekRange{
+		Prefix:    key[:prefixLen],
+		Start:     key[prefixLen : prefixLen+8],
+		Backwards: true,
+	}, func(k, v []byte) bool {
+		lg := &state.TokenTransferLog{Raw: v}
+		cont, err := lg.ForEachNEP11(f)
+		if err != nil {
+			seekErr = err
+		}
+		return cont
+	})
+	return seekErr
+}
+
 // GetTokenTransferLog retrieves transfer log from the cache.
-func (dao *Simple) GetTokenTransferLog(acc util.Uint160, index uint32, isNEP11 bool) (*state.TokenTransferLog, error) {
+func (dao *Simple) GetTokenTransferLog(acc util.Uint160, newestTimestamp uint64, index uint32, isNEP11 bool) (*state.TokenTransferLog, error) {
-	key := getTokenTransferLogKey(acc, index, isNEP11)
+	key := getTokenTransferLogKey(acc, newestTimestamp, index, isNEP11)
 	value, err := dao.Store.Get(key)
 	if err != nil {
 		if err == storage.ErrKeyNotFound {
@@ -206,8 +251,8 @@ func (dao *Simple) GetTokenTransferLog(acc util.Uint160, index uint32, isNEP11 b
 }
 
 // PutTokenTransferLog saves given transfer log in the cache.
-func (dao *Simple) PutTokenTransferLog(acc util.Uint160, index uint32, isNEP11 bool, lg *state.TokenTransferLog) error {
+func (dao *Simple) PutTokenTransferLog(acc util.Uint160, start uint64, index uint32, isNEP11 bool, lg *state.TokenTransferLog) error {
-	key := getTokenTransferLogKey(acc, index, isNEP11)
+	key := getTokenTransferLogKey(acc, start, index, isNEP11)
 	return dao.Store.Put(key, lg.Raw)
 }
 

pkg/core/state/tokens.go

@@ -53,6 +53,10 @@ type TokenTransferInfo struct {
 	NextNEP11Batch uint32
 	// NextNEP17Batch stores the index of the next NEP-17 transfer batch.
 	NextNEP17Batch uint32
+	// NextNEP11NewestTimestamp stores the block timestamp of the first NEP-11 transfer in raw.
+	NextNEP11NewestTimestamp uint64
+	// NextNEP17NewestTimestamp stores the block timestamp of the first NEP-17 transfer in raw.
+	NextNEP17NewestTimestamp uint64
 	// NewNEP11Batch is true if batch with the `NextNEP11Batch` index should be created.
 	NewNEP11Batch bool
 	// NewNEP17Batch is true if batch with the `NextNEP17Batch` index should be created.
@@ -72,6 +76,8 @@ func NewTokenTransferInfo() *TokenTransferInfo {
 func (bs *TokenTransferInfo) DecodeBinary(r *io.BinReader) {
 	bs.NextNEP11Batch = r.ReadU32LE()
 	bs.NextNEP17Batch = r.ReadU32LE()
+	bs.NextNEP11NewestTimestamp = r.ReadU64LE()
+	bs.NextNEP17NewestTimestamp = r.ReadU64LE()
 	bs.NewNEP11Batch = r.ReadBool()
 	bs.NewNEP17Batch = r.ReadBool()
 	lenBalances := r.ReadVarUint()
@@ -87,6 +93,8 @@ func (bs *TokenTransferInfo) DecodeBinary(r *io.BinReader) {
 func (bs *TokenTransferInfo) EncodeBinary(w *io.BinWriter) {
 	w.WriteU32LE(bs.NextNEP11Batch)
 	w.WriteU32LE(bs.NextNEP17Batch)
+	w.WriteU64LE(bs.NextNEP11NewestTimestamp)
+	w.WriteU64LE(bs.NextNEP17NewestTimestamp)
 	w.WriteBool(bs.NewNEP11Batch)
 	w.WriteBool(bs.NewNEP17Batch)
 	w.WriteVarUint(uint64(len(bs.LastUpdated)))

pkg/rpc/server/server.go

@@ -1019,7 +1019,7 @@ func (s *Server) getTokenTransfers(ps request.Params, isNEP11 bool) (interface{}
 		return received, sent, !(limit != 0 && resCount >= limit), nil
 	}
 	if !isNEP11 {
-		err = s.chain.ForEachNEP17Transfer(u, func(tr *state.NEP17Transfer) (bool, error) {
+		err = s.chain.ForEachNEP17Transfer(u, end, func(tr *state.NEP17Transfer) (bool, error) {
 			r, s, res, err := handleTransfer(tr)
 			if err == nil {
 				if r != nil {
@@ -1032,7 +1032,7 @@ func (s *Server) getTokenTransfers(ps request.Params, isNEP11 bool) (interface{}
 			return res, err
 		})
 	} else {
-		err = s.chain.ForEachNEP11Transfer(u, func(tr *state.NEP11Transfer) (bool, error) {
+		err = s.chain.ForEachNEP11Transfer(u, end, func(tr *state.NEP11Transfer) (bool, error) {
 			r, s, res, err := handleTransfer(&tr.NEP17Transfer)
 			if err == nil {
 				id := hex.EncodeToString(tr.ID)
@@ -1047,7 +1047,7 @@ func (s *Server) getTokenTransfers(ps request.Params, isNEP11 bool) (interface{}
 		})
 	}
 	if err != nil {
-		return nil, response.NewInternalServerError("invalid transfer log", err)
+		return nil, response.NewInternalServerError(fmt.Sprintf("invalid transfer log: %v", err), err)
 	}
 	return bs, nil
 }