9b841b9b8f
The results are controversial a bit. MemoryPS is a special storage and the new approach doesn't help it to iterate through NEP17 transfers. However, on long distances (if transfers are requested for more than 1000 blocks, which is ~4.1 hours of mainnet chain) both LevelDB and BoltDB perform good enough with the new approach. Taking into account the fact that default value for the query period of `getnep17transfers` RPC handler is a week (~40K mainnet blocks) we can sacrifice MemoryPS performance in favour of BoltDB and LevelDB optimisation. Close #2263. Benchmark results: name old time/op new time/op delta Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take100Blocks-8 783µs ±13% 1762µs ± 4% +125.12% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take1000Blocks-8 6.91ms ± 2% 9.00ms ± 2% +30.28% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take100Blocks-8 1.43ms ± 8% 1.79ms ± 4% +24.93% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take1000Blocks-8 7.78ms ± 3% 8.93ms ± 2% +14.78% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take100Blocks-8 7.69ms ± 3% 1.73ms ±10% -77.50% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take1000Blocks-8 14.1ms ± 2% 9.0ms ± 2% -36.53% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take100Blocks-8 768µs ± 3% 801µs ± 2% +4.24% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take1000Blocks-8 8.03ms ± 2% 8.05ms ± 8% ~ (p=0.436 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take100Blocks-8 1.70ms ±16% 0.85ms ± 7% -49.85% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take1000Blocks-8 10.8ms ± 2% 8.1ms ± 2% -25.21% (p=0.000 n=10+8) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take100Blocks-8 10.8ms ± 2% 0.9ms ± 7% -92.12% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take1000Blocks-8 18.2ms ±13% 8.2ms ± 6% -54.95% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take100Blocks-8 874µs ± 6% 823µs ± 2% -5.81% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take1000Blocks-8 9.35ms ± 2% 8.14ms ± 2% -12.91% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take100Blocks-8 1.85ms ± 4% 0.89ms ±10% -51.68% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take1000Blocks-8 10.6ms ± 4% 8.2ms ± 2% -22.44% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take100Blocks-8 10.7ms ± 2% 0.9ms ± 2% -91.90% (p=0.000 n=8+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take1000Blocks-8 21.8ms ±15% 8.2ms ± 1% -62.25% (p=0.000 n=10+10) name old alloc/op new alloc/op delta Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take100Blocks-8 650kB ± 0% 1010kB ± 7% +55.45% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take1000Blocks-8 6.39MB ± 0% 9.89MB ± 1% +54.74% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take100Blocks-8 1.28MB ± 0% 1.14MB ± 0% -11.25% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take1000Blocks-8 7.02MB ± 0% 10.01MB ± 0% +42.54% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take100Blocks-8 7.02MB ± 0% 1.08MB ± 0% -84.64% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take1000Blocks-8 12.8MB ± 0% 10.0MB ± 0% -22.04% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take100Blocks-8 823kB ± 3% 866kB ± 5% +5.19% (p=0.001 n=9+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take1000Blocks-8 9.92MB ± 0% 9.85MB ± 0% -0.71% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take100Blocks-8 1.84MB ± 4% 1.01MB ± 5% -44.75% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take1000Blocks-8 11.0MB ± 0% 10.0MB ± 1% -8.45% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take100Blocks-8 11.0MB ± 0% 1.0MB ± 1% -90.55% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take1000Blocks-8 20.0MB ± 0% 10.1MB ± 0% -49.69% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take100Blocks-8 913kB ± 5% 907kB ± 6% ~ (p=0.497 n=9+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take1000Blocks-8 10.0MB ± 1% 10.0MB ± 1% -0.63% (p=0.001 n=10+8) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take100Blocks-8 1.92MB ± 2% 1.04MB ± 0% -45.53% (p=0.000 n=9+8) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take1000Blocks-8 11.1MB ± 1% 10.1MB ± 0% -9.22% (p=0.000 n=10+7) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take100Blocks-8 11.1MB ± 1% 1.0MB ± 0% -90.61% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take1000Blocks-8 20.4MB ± 1% 10.1MB ± 0% -50.46% (p=0.000 n=10+10) name old allocs/op new allocs/op delta Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take100Blocks-8 11.1k ± 0% 11.7k ± 0% +5.35% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1_Take1000Blocks-8 110k ± 0% 110k ± 0% +0.55% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take100Blocks-8 22.0k ± 0% 11.9k ± 0% -46.14% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-100_Take1000Blocks-8 120k ± 0% 110k ± 0% -8.44% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take100Blocks-8 120k ± 0% 12k ± 0% -90.36% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/MemPS_StartFromBlockN-1000_Take1000Blocks-8 219k ± 0% 110k ± 0% -49.73% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take100Blocks-8 11.3k ± 0% 11.2k ± 0% -1.06% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1_Take1000Blocks-8 112k ± 0% 110k ± 0% -2.36% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take100Blocks-8 22.5k ± 0% 11.3k ± 0% -49.65% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-100_Take1000Blocks-8 124k ± 0% 110k ± 0% -11.08% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take100Blocks-8 124k ± 0% 11k ± 0% -90.84% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/BoltPS_StartFromBlockN-1000_Take1000Blocks-8 225k ± 0% 110k ± 0% -51.07% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take100Blocks-8 11.4k ± 0% 11.3k ± 0% -1.36% (p=0.000 n=9+9) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1_Take1000Blocks-8 114k ± 0% 111k ± 0% -2.33% (p=0.000 n=10+9) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take100Blocks-8 22.7k ± 0% 11.5k ± 0% -49.56% (p=0.000 n=9+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-100_Take1000Blocks-8 125k ± 0% 111k ± 0% -11.12% (p=0.000 n=10+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take100Blocks-8 125k ± 0% 11k ± 0% -90.82% (p=0.000 n=8+10) Blockchain_ForEachNEP17Transfer/LevelPS_StartFromBlockN-1000_Take1000Blocks-8 229k ± 1% 111k ± 0% -51.42% (p=0.000 n=10+10)
212 lines
5.9 KiB
Go
212 lines
5.9 KiB
Go
package state
|
|
|
|
import (
|
|
"bytes"
|
|
"math/big"
|
|
|
|
"github.com/nspcc-dev/neo-go/pkg/core/storage"
|
|
"github.com/nspcc-dev/neo-go/pkg/encoding/bigint"
|
|
"github.com/nspcc-dev/neo-go/pkg/io"
|
|
"github.com/nspcc-dev/neo-go/pkg/util"
|
|
)
|
|
|
|
// TokenTransferBatchSize is the maximum number of entries for TokenTransferLog.
|
|
const TokenTransferBatchSize = 128
|
|
|
|
// TokenTransferLog is a serialized log of token transfers.
|
|
type TokenTransferLog struct {
|
|
Raw []byte
|
|
}
|
|
|
|
// NEP17Transfer represents a single NEP-17 Transfer event.
|
|
type NEP17Transfer struct {
|
|
// Asset is a NEP-17 contract ID.
|
|
Asset int32
|
|
// Address is the address of the sender.
|
|
From util.Uint160
|
|
// To is the address of the receiver.
|
|
To util.Uint160
|
|
// Amount is the amount of tokens transferred.
|
|
// It is negative when tokens are sent and positive if they are received.
|
|
Amount big.Int
|
|
// Block is a number of block when the event occurred.
|
|
Block uint32
|
|
// Timestamp is the timestamp of the block where transfer occurred.
|
|
Timestamp uint64
|
|
// Tx is a hash the transaction.
|
|
Tx util.Uint256
|
|
}
|
|
|
|
// NEP11Transfer represents a single NEP-11 Transfer event.
|
|
type NEP11Transfer struct {
|
|
NEP17Transfer
|
|
|
|
// ID is a NEP-11 token ID.
|
|
ID []byte
|
|
}
|
|
|
|
// TokenTransferInfo stores map of the contract IDs to the balance's last updated
|
|
// block trackers along with information about NEP-17 and NEP-11 transfer batch.
|
|
type TokenTransferInfo struct {
|
|
LastUpdated map[int32]uint32
|
|
// NextNEP11Batch stores the index of the next NEP-17 transfer batch.
|
|
NextNEP11Batch uint32
|
|
// NextNEP17Batch stores the index of the next NEP-17 transfer batch.
|
|
NextNEP17Batch uint32
|
|
// NextNEP11NewestTimestamp stores the block timestamp of the first NEP-11 transfer in raw.
|
|
NextNEP11NewestTimestamp uint64
|
|
// NextNEP17NewestTimestamp stores the block timestamp of the first NEP-17 transfer in raw.
|
|
NextNEP17NewestTimestamp uint64
|
|
// NewNEP11Batch is true if batch with the `NextNEP11Batch` index should be created.
|
|
NewNEP11Batch bool
|
|
// NewNEP17Batch is true if batch with the `NextNEP17Batch` index should be created.
|
|
NewNEP17Batch bool
|
|
}
|
|
|
|
// NewTokenTransferInfo returns new TokenTransferInfo.
|
|
func NewTokenTransferInfo() *TokenTransferInfo {
|
|
return &TokenTransferInfo{
|
|
NewNEP11Batch: true,
|
|
NewNEP17Batch: true,
|
|
LastUpdated: make(map[int32]uint32),
|
|
}
|
|
}
|
|
|
|
// DecodeBinary implements io.Serializable interface.
|
|
func (bs *TokenTransferInfo) DecodeBinary(r *io.BinReader) {
|
|
bs.NextNEP11Batch = r.ReadU32LE()
|
|
bs.NextNEP17Batch = r.ReadU32LE()
|
|
bs.NextNEP11NewestTimestamp = r.ReadU64LE()
|
|
bs.NextNEP17NewestTimestamp = r.ReadU64LE()
|
|
bs.NewNEP11Batch = r.ReadBool()
|
|
bs.NewNEP17Batch = r.ReadBool()
|
|
lenBalances := r.ReadVarUint()
|
|
m := make(map[int32]uint32, lenBalances)
|
|
for i := 0; i < int(lenBalances); i++ {
|
|
key := int32(r.ReadU32LE())
|
|
m[key] = r.ReadU32LE()
|
|
}
|
|
bs.LastUpdated = m
|
|
}
|
|
|
|
// EncodeBinary implements io.Serializable interface.
|
|
func (bs *TokenTransferInfo) EncodeBinary(w *io.BinWriter) {
|
|
w.WriteU32LE(bs.NextNEP11Batch)
|
|
w.WriteU32LE(bs.NextNEP17Batch)
|
|
w.WriteU64LE(bs.NextNEP11NewestTimestamp)
|
|
w.WriteU64LE(bs.NextNEP17NewestTimestamp)
|
|
w.WriteBool(bs.NewNEP11Batch)
|
|
w.WriteBool(bs.NewNEP17Batch)
|
|
w.WriteVarUint(uint64(len(bs.LastUpdated)))
|
|
for k, v := range bs.LastUpdated {
|
|
w.WriteU32LE(uint32(k))
|
|
w.WriteU32LE(v)
|
|
}
|
|
}
|
|
|
|
// Append appends single transfer to a log.
|
|
func (lg *TokenTransferLog) Append(tr io.Serializable) error {
|
|
// The first entry, set up counter.
|
|
if len(lg.Raw) == 0 {
|
|
lg.Raw = append(lg.Raw, 0)
|
|
}
|
|
|
|
b := bytes.NewBuffer(lg.Raw)
|
|
w := io.NewBinWriterFromIO(b)
|
|
|
|
tr.EncodeBinary(w)
|
|
if w.Err != nil {
|
|
return w.Err
|
|
}
|
|
lg.Raw = b.Bytes()
|
|
lg.Raw[0]++
|
|
return nil
|
|
}
|
|
|
|
// ForEachNEP11 iterates over transfer log returning on first error.
|
|
func (lg *TokenTransferLog) ForEachNEP11(f func(*NEP11Transfer) (bool, error)) (bool, error) {
|
|
if lg == nil || len(lg.Raw) == 0 {
|
|
return true, nil
|
|
}
|
|
transfers := make([]NEP11Transfer, lg.Size())
|
|
r := io.NewBinReaderFromBuf(lg.Raw[1:])
|
|
for i := 0; i < lg.Size(); i++ {
|
|
transfers[i].DecodeBinary(r)
|
|
}
|
|
if r.Err != nil {
|
|
return false, r.Err
|
|
}
|
|
for i := len(transfers) - 1; i >= 0; i-- {
|
|
cont, err := f(&transfers[i])
|
|
if err != nil || !cont {
|
|
return false, err
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// ForEachNEP17 iterates over transfer log returning on first error.
|
|
func (lg *TokenTransferLog) ForEachNEP17(f func(*NEP17Transfer) (bool, error)) (bool, error) {
|
|
if lg == nil || len(lg.Raw) == 0 {
|
|
return true, nil
|
|
}
|
|
transfers := make([]NEP17Transfer, lg.Size())
|
|
r := io.NewBinReaderFromBuf(lg.Raw[1:])
|
|
for i := 0; i < lg.Size(); i++ {
|
|
transfers[i].DecodeBinary(r)
|
|
}
|
|
if r.Err != nil {
|
|
return false, r.Err
|
|
}
|
|
for i := len(transfers) - 1; i >= 0; i-- {
|
|
cont, err := f(&transfers[i])
|
|
if err != nil || !cont {
|
|
return false, err
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// Size returns an amount of transfer written in log.
|
|
func (lg *TokenTransferLog) Size() int {
|
|
if len(lg.Raw) == 0 {
|
|
return 0
|
|
}
|
|
return int(lg.Raw[0])
|
|
}
|
|
|
|
// EncodeBinary implements io.Serializable interface.
|
|
func (t *NEP17Transfer) EncodeBinary(w *io.BinWriter) {
|
|
w.WriteU32LE(uint32(t.Asset))
|
|
w.WriteBytes(t.Tx[:])
|
|
w.WriteBytes(t.From[:])
|
|
w.WriteBytes(t.To[:])
|
|
w.WriteU32LE(t.Block)
|
|
w.WriteU64LE(t.Timestamp)
|
|
amount := bigint.ToBytes(&t.Amount)
|
|
w.WriteVarBytes(amount)
|
|
}
|
|
|
|
// DecodeBinary implements io.Serializable interface.
|
|
func (t *NEP17Transfer) DecodeBinary(r *io.BinReader) {
|
|
t.Asset = int32(r.ReadU32LE())
|
|
r.ReadBytes(t.Tx[:])
|
|
r.ReadBytes(t.From[:])
|
|
r.ReadBytes(t.To[:])
|
|
t.Block = r.ReadU32LE()
|
|
t.Timestamp = r.ReadU64LE()
|
|
amount := r.ReadVarBytes(bigint.MaxBytesLen)
|
|
t.Amount = *bigint.FromBytes(amount)
|
|
}
|
|
|
|
// EncodeBinary implements io.Serializable interface.
|
|
func (t *NEP11Transfer) EncodeBinary(w *io.BinWriter) {
|
|
t.NEP17Transfer.EncodeBinary(w)
|
|
w.WriteVarBytes(t.ID)
|
|
}
|
|
|
|
// DecodeBinary implements io.Serializable interface.
|
|
func (t *NEP11Transfer) DecodeBinary(r *io.BinReader) {
|
|
t.NEP17Transfer.DecodeBinary(r)
|
|
t.ID = r.ReadVarBytes(storage.MaxStorageKeyLen)
|
|
}
|