neoneo-go/pkg/core/state/tokens.go
Anna Shaleva 9b841b9b8f core, rpc: use Seek to iterate over NEP* transfers
The results are controversial a bit. MemoryPS is a special storage
and the new approach doesn't help it to iterate through NEP17 transfers.
However, on long distances (if transfers are requested for more than
1000 blocks, which is ~4.1 hours of mainnet chain) both LevelDB and
BoltDB perform good enough with the new approach. Taking into account
the fact that default value for the query period of `getnep17transfers`
RPC handler is a week (~40K mainnet blocks) we can sacrifice MemoryPS
performance in favour of BoltDB and LevelDB optimisation.

Close #2263.

Benchmark results:

name                                                                           old time/op    new time/op    delta
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take100Blocks-8           783µs ±13%    1762µs ± 4%  +125.12%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take1000Blocks-8         6.91ms ± 2%    9.00ms ± 2%   +30.28%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take100Blocks-8        1.43ms ± 8%    1.79ms ± 4%   +24.93%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take1000Blocks-8       7.78ms ± 3%    8.93ms ± 2%   +14.78%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take100Blocks-8       7.69ms ± 3%    1.73ms ±10%   -77.50%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take1000Blocks-8      14.1ms ± 2%     9.0ms ± 2%   -36.53%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take100Blocks-8          768µs ± 3%     801µs ± 2%    +4.24%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take1000Blocks-8        8.03ms ± 2%    8.05ms ± 8%      ~     (p=0.436 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take100Blocks-8       1.70ms ±16%    0.85ms ± 7%   -49.85%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take1000Blocks-8      10.8ms ± 2%     8.1ms ± 2%   -25.21%  (p=0.000 n=10+8)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take100Blocks-8      10.8ms ± 2%     0.9ms ± 7%   -92.12%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take1000Blocks-8     18.2ms ±13%     8.2ms ± 6%   -54.95%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take100Blocks-8         874µs ± 6%     823µs ± 2%    -5.81%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take1000Blocks-8       9.35ms ± 2%    8.14ms ± 2%   -12.91%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take100Blocks-8      1.85ms ± 4%    0.89ms ±10%   -51.68%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take1000Blocks-8     10.6ms ± 4%     8.2ms ± 2%   -22.44%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take100Blocks-8     10.7ms ± 2%     0.9ms ± 2%   -91.90%  (p=0.000 n=8+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take1000Blocks-8    21.8ms ±15%     8.2ms ± 1%   -62.25%  (p=0.000 n=10+10)

name                                                                           old alloc/op   new alloc/op   delta
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take100Blocks-8           650kB ± 0%    1010kB ± 7%   +55.45%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take1000Blocks-8         6.39MB ± 0%    9.89MB ± 1%   +54.74%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take100Blocks-8        1.28MB ± 0%    1.14MB ± 0%   -11.25%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take1000Blocks-8       7.02MB ± 0%   10.01MB ± 0%   +42.54%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take100Blocks-8       7.02MB ± 0%    1.08MB ± 0%   -84.64%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take1000Blocks-8      12.8MB ± 0%    10.0MB ± 0%   -22.04%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take100Blocks-8          823kB ± 3%     866kB ± 5%    +5.19%  (p=0.001 n=9+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take1000Blocks-8        9.92MB ± 0%    9.85MB ± 0%    -0.71%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take100Blocks-8       1.84MB ± 4%    1.01MB ± 5%   -44.75%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take1000Blocks-8      11.0MB ± 0%    10.0MB ± 1%    -8.45%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take100Blocks-8      11.0MB ± 0%     1.0MB ± 1%   -90.55%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take1000Blocks-8     20.0MB ± 0%    10.1MB ± 0%   -49.69%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take100Blocks-8         913kB ± 5%     907kB ± 6%      ~     (p=0.497 n=9+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take1000Blocks-8       10.0MB ± 1%    10.0MB ± 1%    -0.63%  (p=0.001 n=10+8)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take100Blocks-8      1.92MB ± 2%    1.04MB ± 0%   -45.53%  (p=0.000 n=9+8)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take1000Blocks-8     11.1MB ± 1%    10.1MB ± 0%    -9.22%  (p=0.000 n=10+7)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take100Blocks-8     11.1MB ± 1%     1.0MB ± 0%   -90.61%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take1000Blocks-8    20.4MB ± 1%    10.1MB ± 0%   -50.46%  (p=0.000 n=10+10)

name                                                                           old allocs/op  new allocs/op  delta
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take100Blocks-8           11.1k ± 0%     11.7k ± 0%    +5.35%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take1000Blocks-8           110k ± 0%      110k ± 0%    +0.55%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take100Blocks-8         22.0k ± 0%     11.9k ± 0%   -46.14%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take1000Blocks-8         120k ± 0%      110k ± 0%    -8.44%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take100Blocks-8         120k ± 0%       12k ± 0%   -90.36%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take1000Blocks-8        219k ± 0%      110k ± 0%   -49.73%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take100Blocks-8          11.3k ± 0%     11.2k ± 0%    -1.06%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take1000Blocks-8          112k ± 0%      110k ± 0%    -2.36%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take100Blocks-8        22.5k ± 0%     11.3k ± 0%   -49.65%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take1000Blocks-8        124k ± 0%      110k ± 0%   -11.08%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take100Blocks-8        124k ± 0%       11k ± 0%   -90.84%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take1000Blocks-8       225k ± 0%      110k ± 0%   -51.07%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take100Blocks-8         11.4k ± 0%     11.3k ± 0%    -1.36%  (p=0.000 n=9+9)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take1000Blocks-8         114k ± 0%      111k ± 0%    -2.33%  (p=0.000 n=10+9)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take100Blocks-8       22.7k ± 0%     11.5k ± 0%   -49.56%  (p=0.000 n=9+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take1000Blocks-8       125k ± 0%      111k ± 0%   -11.12%  (p=0.000 n=10+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take100Blocks-8       125k ± 0%       11k ± 0%   -90.82%  (p=0.000 n=8+10)
Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take1000Blocks-8      229k ± 1%      111k ± 0%   -51.42%  (p=0.000 n=10+10)
2022-01-19 20:55:08 +03:00

212 lines
5.9 KiB
Go

package state
import (
"bytes"
"math/big"
"github.com/nspcc-dev/neo-go/pkg/core/storage"
"github.com/nspcc-dev/neo-go/pkg/encoding/bigint"
"github.com/nspcc-dev/neo-go/pkg/io"
"github.com/nspcc-dev/neo-go/pkg/util"
)
// TokenTransferBatchSize is the maximum number of entries for TokenTransferLog.
const TokenTransferBatchSize = 128
// TokenTransferLog is a serialized log of token transfers.
type TokenTransferLog struct {
Raw []byte
}
// NEP17Transfer represents a single NEP-17 Transfer event.
type NEP17Transfer struct {
// Asset is a NEP-17 contract ID.
Asset int32
// Address is the address of the sender.
From util.Uint160
// To is the address of the receiver.
To util.Uint160
// Amount is the amount of tokens transferred.
// It is negative when tokens are sent and positive if they are received.
Amount big.Int
// Block is a number of block when the event occurred.
Block uint32
// Timestamp is the timestamp of the block where transfer occurred.
Timestamp uint64
// Tx is a hash the transaction.
Tx util.Uint256
}
// NEP11Transfer represents a single NEP-11 Transfer event.
type NEP11Transfer struct {
NEP17Transfer
// ID is a NEP-11 token ID.
ID []byte
}
// TokenTransferInfo stores map of the contract IDs to the balance's last updated
// block trackers along with information about NEP-17 and NEP-11 transfer batch.
type TokenTransferInfo struct {
LastUpdated map[int32]uint32
// NextNEP11Batch stores the index of the next NEP-17 transfer batch.
NextNEP11Batch uint32
// NextNEP17Batch stores the index of the next NEP-17 transfer batch.
NextNEP17Batch uint32
// NextNEP11NewestTimestamp stores the block timestamp of the first NEP-11 transfer in raw.
NextNEP11NewestTimestamp uint64
// NextNEP17NewestTimestamp stores the block timestamp of the first NEP-17 transfer in raw.
NextNEP17NewestTimestamp uint64
// NewNEP11Batch is true if batch with the `NextNEP11Batch` index should be created.
NewNEP11Batch bool
// NewNEP17Batch is true if batch with the `NextNEP17Batch` index should be created.
NewNEP17Batch bool
}
// NewTokenTransferInfo returns new TokenTransferInfo.
func NewTokenTransferInfo() *TokenTransferInfo {
return &TokenTransferInfo{
NewNEP11Batch: true,
NewNEP17Batch: true,
LastUpdated: make(map[int32]uint32),
}
}
// DecodeBinary implements io.Serializable interface.
func (bs *TokenTransferInfo) DecodeBinary(r *io.BinReader) {
bs.NextNEP11Batch = r.ReadU32LE()
bs.NextNEP17Batch = r.ReadU32LE()
bs.NextNEP11NewestTimestamp = r.ReadU64LE()
bs.NextNEP17NewestTimestamp = r.ReadU64LE()
bs.NewNEP11Batch = r.ReadBool()
bs.NewNEP17Batch = r.ReadBool()
lenBalances := r.ReadVarUint()
m := make(map[int32]uint32, lenBalances)
for i := 0; i < int(lenBalances); i++ {
key := int32(r.ReadU32LE())
m[key] = r.ReadU32LE()
}
bs.LastUpdated = m
}
// EncodeBinary implements io.Serializable interface.
func (bs *TokenTransferInfo) EncodeBinary(w *io.BinWriter) {
w.WriteU32LE(bs.NextNEP11Batch)
w.WriteU32LE(bs.NextNEP17Batch)
w.WriteU64LE(bs.NextNEP11NewestTimestamp)
w.WriteU64LE(bs.NextNEP17NewestTimestamp)
w.WriteBool(bs.NewNEP11Batch)
w.WriteBool(bs.NewNEP17Batch)
w.WriteVarUint(uint64(len(bs.LastUpdated)))
for k, v := range bs.LastUpdated {
w.WriteU32LE(uint32(k))
w.WriteU32LE(v)
}
}
// Append appends single transfer to a log.
func (lg *TokenTransferLog) Append(tr io.Serializable) error {
// The first entry, set up counter.
if len(lg.Raw) == 0 {
lg.Raw = append(lg.Raw, 0)
}
b := bytes.NewBuffer(lg.Raw)
w := io.NewBinWriterFromIO(b)
tr.EncodeBinary(w)
if w.Err != nil {
return w.Err
}
lg.Raw = b.Bytes()
lg.Raw[0]++
return nil
}
// ForEachNEP11 iterates over transfer log returning on first error.
func (lg *TokenTransferLog) ForEachNEP11(f func(*NEP11Transfer) (bool, error)) (bool, error) {
if lg == nil || len(lg.Raw) == 0 {
return true, nil
}
transfers := make([]NEP11Transfer, lg.Size())
r := io.NewBinReaderFromBuf(lg.Raw[1:])
for i := 0; i < lg.Size(); i++ {
transfers[i].DecodeBinary(r)
}
if r.Err != nil {
return false, r.Err
}
for i := len(transfers) - 1; i >= 0; i-- {
cont, err := f(&transfers[i])
if err != nil || !cont {
return false, err
}
}
return true, nil
}
// ForEachNEP17 iterates over transfer log returning on first error.
func (lg *TokenTransferLog) ForEachNEP17(f func(*NEP17Transfer) (bool, error)) (bool, error) {
if lg == nil || len(lg.Raw) == 0 {
return true, nil
}
transfers := make([]NEP17Transfer, lg.Size())
r := io.NewBinReaderFromBuf(lg.Raw[1:])
for i := 0; i < lg.Size(); i++ {
transfers[i].DecodeBinary(r)
}
if r.Err != nil {
return false, r.Err
}
for i := len(transfers) - 1; i >= 0; i-- {
cont, err := f(&transfers[i])
if err != nil || !cont {
return false, err
}
}
return true, nil
}
// Size returns an amount of transfer written in log.
func (lg *TokenTransferLog) Size() int {
if len(lg.Raw) == 0 {
return 0
}
return int(lg.Raw[0])
}
// EncodeBinary implements io.Serializable interface.
func (t *NEP17Transfer) EncodeBinary(w *io.BinWriter) {
w.WriteU32LE(uint32(t.Asset))
w.WriteBytes(t.Tx[:])
w.WriteBytes(t.From[:])
w.WriteBytes(t.To[:])
w.WriteU32LE(t.Block)
w.WriteU64LE(t.Timestamp)
amount := bigint.ToBytes(&t.Amount)
w.WriteVarBytes(amount)
}
// DecodeBinary implements io.Serializable interface.
func (t *NEP17Transfer) DecodeBinary(r *io.BinReader) {
t.Asset = int32(r.ReadU32LE())
r.ReadBytes(t.Tx[:])
r.ReadBytes(t.From[:])
r.ReadBytes(t.To[:])
t.Block = r.ReadU32LE()
t.Timestamp = r.ReadU64LE()
amount := r.ReadVarBytes(bigint.MaxBytesLen)
t.Amount = *bigint.FromBytes(amount)
}
// EncodeBinary implements io.Serializable interface.
func (t *NEP11Transfer) EncodeBinary(w *io.BinWriter) {
t.NEP17Transfer.EncodeBinary(w)
w.WriteVarBytes(t.ID)
}
// DecodeBinary implements io.Serializable interface.
func (t *NEP11Transfer) DecodeBinary(r *io.BinReader) {
t.NEP17Transfer.DecodeBinary(r)
t.ID = r.ReadVarBytes(storage.MaxStorageKeyLen)
}