core: implement dynamic NEP17 balances tracking

Request NEP17 balances from a set of NEP17 contracts instead of getting
them from storage. LastUpdatedBlock tracking remains untouched, because
there's no way to retrieve it dynamically.

docs/rpc.md

@@ -97,6 +97,25 @@ it only works for native contracts.
 This method doesn't work for the Ledger contract, you can get data via regular
 `getblock` and `getrawtransaction` calls.
 
+#### `getnep17balances`
+
+neo-go's implementation of `getnep17balances` does not perform tracking of NEP17
+balances for each account as it is done in the C# node. Instead, neo-go node
+maintains the list of NEP17-compliant contracts, i.e. those contracts that have
+`NEP-17` declared in the supported standards section of the manifest. Each time
+`getnep17balances` is queried, neo-go node asks every NEP17 contract for the
+account balance by invoking `balanceOf` method with the corresponding args.
+Invocation GAS limit is set to be 3 GAS. All non-zero NEP17 balances are included
+in the RPC call result.
+
+Thus, if NEP17 token contract doesn't have `NEP-17` standard declared in the list
+of supported standards but emits proper NEP17 `Transfer` notifications, the token
+balance won't be shown in the list of NEP17 balances returned by the neo-go node
+(unlike the C# node behavior). However, transfer logs of such token are still
+available via `getnep17transfers` RPC call.
+
+The behaviour of the `LastUpdatedBlock` tracking matches the C# node's one.
+
 ### Unsupported methods
 
 Methods listed down below are not going to be supported for various reasons

internal/fakechain/fakechain.go

@@ -254,13 +254,18 @@ func (chain *FakeChain) GetNextBlockValidators() ([]*keys.PublicKey, error) {
 	panic("TODO")
 }
 
-// ForEachNEP17Transfer implements Blockchainer interface.
+// GetNEP17Contracts implements Blockchainer interface.
-func (chain *FakeChain) ForEachNEP17Transfer(util.Uint160, func(*state.NEP17Transfer) (bool, error)) error {
+func (chain *FakeChain) GetNEP17Contracts() []util.Uint160 {
 	panic("TODO")
 }
 
-// GetNEP17Balances implements Blockchainer interface.
+// GetNEP17LastUpdated implements Blockchainer interface.
-func (chain *FakeChain) GetNEP17Balances(util.Uint160) *state.NEP17TransferInfo {
+func (chain *FakeChain) GetNEP17LastUpdated(acc util.Uint160) (map[int32]uint32, error) {
+	panic("TODO")
+}
+
+// ForEachNEP17Transfer implements Blockchainer interface.
+func (chain *FakeChain) ForEachNEP17Transfer(util.Uint160, func(*state.NEP17Transfer) (bool, error)) error {
 	panic("TODO")
 }
 

pkg/core/blockchain.go

@@ -52,7 +52,8 @@ const (
 	defaultMaxBlockSystemFee               = 900000000000
 	defaultMaxTraceableBlocks              = 2102400 // 1 year of 15s blocks
 	defaultMaxTransactionsPerBlock         = 512
-	headerVerificationGasLimit             = 3_00000000 // 3 GAS
+	// HeaderVerificationGasLimit is the maximum amount of GAS for block header verification.
+	HeaderVerificationGasLimit = 3_00000000 // 3 GAS
 )
 
 var (
@@ -993,10 +994,7 @@ func (bc *Blockchain) processNEP17Transfer(cache *dao.Cached, h util.Uint256, b
 		if err != nil {
 			return
 		}
-		bs := balances.LastUpdated[id]
+		balances.LastUpdated[id] = b.Index
-		bs.Balance = *new(big.Int).Sub(&bs.Balance, amount)
-		bs.LastUpdatedBlock = b.Index
-		balances.LastUpdated[id] = bs
 		transfer.Amount = *new(big.Int).Sub(&transfer.Amount, amount)
 		balances.NewBatch, err = cache.AppendNEP17Transfer(fromAddr,
 			balances.NextTransferBatch, balances.NewBatch, transfer)
@@ -1015,10 +1013,7 @@ func (bc *Blockchain) processNEP17Transfer(cache *dao.Cached, h util.Uint256, b
 		if err != nil {
 			return
 		}
-		bs := balances.LastUpdated[id]
+		balances.LastUpdated[id] = b.Index
-		bs.Balance = *new(big.Int).Add(&bs.Balance, amount)
-		bs.LastUpdatedBlock = b.Index
-		balances.LastUpdated[id] = bs
 
 		transfer.Amount = *amount
 		balances.NewBatch, err = cache.AppendNEP17Transfer(toAddr,
@@ -1057,34 +1052,34 @@ func (bc *Blockchain) ForEachNEP17Transfer(acc util.Uint160, f func(*state.NEP17
 	return nil
 }
 
-// GetNEP17Balances returns NEP17 balances for the acc.
+// GetNEP17Contracts returns the list of deployed NEP17 contracts.
-func (bc *Blockchain) GetNEP17Balances(acc util.Uint160) *state.NEP17TransferInfo {
+func (bc *Blockchain) GetNEP17Contracts() []util.Uint160 {
-	bs, err := bc.dao.GetNEP17TransferInfo(acc)
+	return bc.contracts.Management.GetNEP17Contracts()
-	if err != nil {
-		return nil
 }
-	return bs
+
+// GetNEP17LastUpdated returns a set of contract ids with the corresponding last updated
+// block indexes.
+func (bc *Blockchain) GetNEP17LastUpdated(acc util.Uint160) (map[int32]uint32, error) {
+	info, err := bc.dao.GetNEP17TransferInfo(acc)
+	if err != nil {
+		return nil, err
+	}
+	return info.LastUpdated, nil
 }
 
 // GetUtilityTokenBalance returns utility token (GAS) balance for the acc.
 func (bc *Blockchain) GetUtilityTokenBalance(acc util.Uint160) *big.Int {
-	bs, err := bc.dao.GetNEP17TransferInfo(acc)
+	bs := bc.contracts.GAS.BalanceOf(bc.dao, acc)
-	if err != nil {
+	if bs == nil {
 		return big.NewInt(0)
 	}
-	balance := bs.LastUpdated[bc.contracts.GAS.ID].Balance
+	return bs
-	return &balance
 }
 
 // GetGoverningTokenBalance returns governing token (NEO) balance and the height
 // of the last balance change for the account.
 func (bc *Blockchain) GetGoverningTokenBalance(acc util.Uint160) (*big.Int, uint32) {
-	bs, err := bc.dao.GetNEP17TransferInfo(acc)
+	return bc.contracts.NEO.BalanceOf(bc.dao, acc)
-	if err != nil {
-		return big.NewInt(0), 0
-	}
-	neo := bs.LastUpdated[bc.contracts.NEO.ID]
-	return &neo.Balance, neo.LastUpdatedBlock
 }
 
 // GetNotaryBalance returns Notary deposit amount for the specified account.
@@ -1873,7 +1868,7 @@ func (bc *Blockchain) verifyHeaderWitnesses(currHeader, prevHeader *block.Header
 	} else {
 		hash = prevHeader.NextConsensus
 	}
-	return bc.VerifyWitness(hash, currHeader, &currHeader.Script, headerVerificationGasLimit)
+	return bc.VerifyWitness(hash, currHeader, &currHeader.Script, HeaderVerificationGasLimit)
 }
 
 // GoverningTokenHash returns the governing token (NEO) native contract hash.

pkg/core/blockchainer/blockchainer.go

@@ -47,7 +47,8 @@ type Blockchainer interface {
 	GetNativeContractScriptHash(string) (util.Uint160, error)
 	GetNatives() []state.NativeContract
 	GetNextBlockValidators() ([]*keys.PublicKey, error)
-	GetNEP17Balances(util.Uint160) *state.NEP17TransferInfo
+	GetNEP17Contracts() []util.Uint160
+	GetNEP17LastUpdated(acc util.Uint160) (map[int32]uint32, error)
 	GetNotaryContractScriptHash() util.Uint160
 	GetNotaryBalance(acc util.Uint160) *big.Int
 	GetPolicer() Policer

pkg/core/helper_test.go

@@ -713,8 +713,8 @@ func checkFAULTState(t *testing.T, result *state.AppExecResult) {
 }
 
 func checkBalanceOf(t *testing.T, chain *Blockchain, addr util.Uint160, expected int) {
-	balance := chain.GetNEP17Balances(addr).LastUpdated[chain.contracts.GAS.ID]
+	balance := chain.GetUtilityTokenBalance(addr)
-	require.Equal(t, int64(expected), balance.Balance.Int64())
+	require.Equal(t, int64(expected), balance.Int64())
 }
 
 type NotaryFeerStub struct {

pkg/core/native/native_gas.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math/big"
 
+	"github.com/nspcc-dev/neo-go/pkg/core/dao"
 	"github.com/nspcc-dev/neo-go/pkg/core/interop"
 	"github.com/nspcc-dev/neo-go/pkg/core/interop/runtime"
 	"github.com/nspcc-dev/neo-go/pkg/core/native/nativenames"
@@ -138,6 +139,11 @@ func (g *GAS) PostPersist(ic *interop.Context) error {
 	return nil
 }
 
+// BalanceOf returns native GAS token balance for the acc.
+func (g *GAS) BalanceOf(d dao.DAO, acc util.Uint160) *big.Int {
+	return g.balanceOfInternal(d, acc)
+}
+
 func getStandbyValidatorsHash(ic *interop.Context) (util.Uint160, error) {
 	s, err := smartcontract.CreateDefaultMultiSigRedeemScript(ic.Chain.GetStandByValidators())
 	if err != nil {

pkg/core/native/native_neo.go

@@ -1020,6 +1020,20 @@ func (n *NEO) GetNextBlockValidatorsInternal() keys.PublicKeys {
 	return n.nextValidators.Load().(keys.PublicKeys).Copy()
 }
 
+// BalanceOf returns native NEO token balance for the acc.
+func (n *NEO) BalanceOf(d dao.DAO, acc util.Uint160) (*big.Int, uint32) {
+	key := makeAccountKey(acc)
+	si := d.GetStorageItem(n.ID, key)
+	if si == nil {
+		return big.NewInt(0), 0
+	}
+	st, err := state.NEOBalanceFromBytes(si)
+	if err != nil {
+		panic(fmt.Errorf("failed to decode NEO balance state: %w", err))
+	}
+	return &st.Balance, st.BalanceHeight
+}
+
 func pubsToArray(pubs keys.PublicKeys) stackitem.Item {
 	arr := make([]stackitem.Item, len(pubs))
 	for i := range pubs {

pkg/core/native/native_nep17.go

@@ -246,16 +246,20 @@ func (c *nep17TokenNative) TransferInternal(ic *interop.Context, from, to util.U
 
 func (c *nep17TokenNative) balanceOf(ic *interop.Context, args []stackitem.Item) stackitem.Item {
 	h := toUint160(args[0])
+	return stackitem.NewBigInteger(c.balanceOfInternal(ic.DAO, h))
+}
+
+func (c *nep17TokenNative) balanceOfInternal(d dao.DAO, h util.Uint160) *big.Int {
 	key := makeAccountKey(h)
-	si := ic.DAO.GetStorageItem(c.ID, key)
+	si := d.GetStorageItem(c.ID, key)
 	if si == nil {
-		return stackitem.NewBigInteger(big.NewInt(0))
+		return big.NewInt(0)
 	}
 	balance, err := c.balFromBytes(&si)
 	if err != nil {
 		panic(fmt.Errorf("can not deserialize balance state: %w", err))
 	}
-	return stackitem.NewBigInteger(balance)
+	return balance
 }
 
 func (c *nep17TokenNative) mint(ic *interop.Context, h util.Uint160, amount *big.Int, callOnPayment bool) {

pkg/core/native_gas_test.go

@@ -76,12 +76,9 @@ func TestGAS_Roundtrip(t *testing.T) {
 	bc := newTestChain(t)
 
 	getUtilityTokenBalance := func(bc *Blockchain, acc util.Uint160) (*big.Int, uint32) {
-		bs, err := bc.dao.GetNEP17TransferInfo(acc)
+		lub, err := bc.GetNEP17LastUpdated(acc)
-		if err != nil {
+		require.NoError(t, err)
-			return big.NewInt(0), 0
+		return bc.GetUtilityTokenBalance(acc), lub[bc.contracts.GAS.ID]
-		}
-		balance := bs.LastUpdated[bc.contracts.GAS.ID]
-		return &balance.Balance, balance.LastUpdatedBlock
 	}
 
 	initialBalance, _ := getUtilityTokenBalance(bc, neoOwner)

pkg/core/state/nep17.go

@@ -11,15 +11,6 @@ import (
 // NEP17TransferBatchSize is the maximum number of entries for NEP17TransferLog.
 const NEP17TransferBatchSize = 128
 
-// NEP17Tracker contains info about a single account in a NEP17 contract.
-type NEP17Tracker struct {
-	// Balance is the current balance of the account.
-	Balance big.Int
-	// LastUpdatedBlock is a number of block when last `transfer` to or from the
-	// account occurred.
-	LastUpdatedBlock uint32
-}
-
 // NEP17TransferLog is a log of NEP17 token transfers for the specific command.
 type NEP17TransferLog struct {
 	Raw []byte
@@ -44,10 +35,10 @@ type NEP17Transfer struct {
 	Tx util.Uint256
 }
 
-// NEP17TransferInfo is a map of the NEP17 contract IDs
+// NEP17TransferInfo stores map of the NEP17 contract IDs to the balance's last updated
-// to the corresponding structures.
+// block trackers along with information about NEP17 transfer batch.
 type NEP17TransferInfo struct {
-	LastUpdated map[int32]NEP17Tracker
+	LastUpdated map[int32]uint32
 	// NextTransferBatch stores an index of the next transfer batch.
 	NextTransferBatch uint32
 	// NewBatch is true if batch with the `NextTransferBatch` index should be created.
@@ -57,7 +48,7 @@ type NEP17TransferInfo struct {
 // NewNEP17TransferInfo returns new NEP17TransferInfo.
 func NewNEP17TransferInfo() *NEP17TransferInfo {
 	return &NEP17TransferInfo{
-		LastUpdated: make(map[int32]NEP17Tracker),
+		LastUpdated: make(map[int32]uint32),
 	}
 }
 
@@ -66,12 +57,10 @@ func (bs *NEP17TransferInfo) DecodeBinary(r *io.BinReader) {
 	bs.NextTransferBatch = r.ReadU32LE()
 	bs.NewBatch = r.ReadBool()
 	lenBalances := r.ReadVarUint()
-	m := make(map[int32]NEP17Tracker, lenBalances)
+	m := make(map[int32]uint32, lenBalances)
 	for i := 0; i < int(lenBalances); i++ {
 		key := int32(r.ReadU32LE())
-		var tr NEP17Tracker
+		m[key] = r.ReadU32LE()
-		tr.DecodeBinary(r)
-		m[key] = tr
 	}
 	bs.LastUpdated = m
 }
@@ -83,7 +72,7 @@ func (bs *NEP17TransferInfo) EncodeBinary(w *io.BinWriter) {
 	w.WriteVarUint(uint64(len(bs.LastUpdated)))
 	for k, v := range bs.LastUpdated {
 		w.WriteU32LE(uint32(k))
-		v.EncodeBinary(w)
+		w.WriteU32LE(v)
 	}
 }
 
@@ -138,18 +127,6 @@ func (lg *NEP17TransferLog) Size() int {
 	return int(lg.Raw[0])
 }
 
-// EncodeBinary implements io.Serializable interface.
-func (t *NEP17Tracker) EncodeBinary(w *io.BinWriter) {
-	w.WriteVarBytes(bigint.ToBytes(&t.Balance))
-	w.WriteU32LE(t.LastUpdatedBlock)
-}
-
-// DecodeBinary implements io.Serializable interface.
-func (t *NEP17Tracker) DecodeBinary(r *io.BinReader) {
-	t.Balance = *bigint.FromBytes(r.ReadVarBytes())
-	t.LastUpdatedBlock = r.ReadU32LE()
-}
-
 // EncodeBinary implements io.Serializable interface.
 func (t *NEP17Transfer) EncodeBinary(w *io.BinWriter) {
 	w.WriteU32LE(uint32(t.Asset))

pkg/core/state/nep17_test.go

@@ -38,15 +38,6 @@ func TestNEP17TransferLog_Append(t *testing.T) {
 	require.True(t, cont)
 }
 
-func TestNEP17Tracker_EncodeBinary(t *testing.T) {
-	expected := &NEP17Tracker{
-		Balance:          *big.NewInt(int64(rand.Uint64())),
-		LastUpdatedBlock: rand.Uint32(),
-	}
-
-	testserdes.EncodeDecodeBinary(t, expected, new(NEP17Tracker))
-}
-
 func TestNEP17Transfer_DecodeBinary(t *testing.T) {
 	expected := &NEP17Transfer{
 		Asset:     123,

pkg/rpc/server/server.go

@@ -40,6 +40,7 @@ import (
 	"github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag"
 	"github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger"
 	"github.com/nspcc-dev/neo-go/pkg/util"
+	"github.com/nspcc-dev/neo-go/pkg/vm/emit"
 	"github.com/nspcc-dev/neo-go/pkg/vm/opcode"
 	"go.uber.org/zap"
 )
@@ -669,28 +670,65 @@ func (s *Server) getNEP17Balances(ps request.Params) (interface{}, *response.Err
 		return nil, response.ErrInvalidParams
 	}
 
-	as := s.chain.GetNEP17Balances(u)
 	bs := &result.NEP17Balances{
 		Address:  address.Uint160ToString(u),
 		Balances: []result.NEP17Balance{},
 	}
-	if as != nil {
+	lastUpdated, err := s.chain.GetNEP17LastUpdated(u)
-		cache := make(map[int32]util.Uint160)
-		for id, bal := range as.LastUpdated {
-			h, err := s.getHash(id, cache)
 	if err != nil {
+		return nil, response.NewRPCError("Failed to get NEP17 last updated block", err.Error(), err)
+	}
+	bw := io.NewBufBinWriter()
+	for _, h := range s.chain.GetNEP17Contracts() {
+		balance, err := s.getNEP17Balance(h, u, bw)
+		if err != nil {
+			continue
+		}
+		if balance.Sign() == 0 {
+			continue
+		}
+		cs := s.chain.GetContractState(h)
+		if cs == nil {
 			continue
 		}
 		bs.Balances = append(bs.Balances, result.NEP17Balance{
 			Asset:       h,
-				Amount:      bal.Balance.String(),
+			Amount:      balance.String(),
-				LastUpdated: bal.LastUpdatedBlock,
+			LastUpdated: lastUpdated[cs.ID],
 		})
 	}
-	}
 	return bs, nil
 }
 
+func (s *Server) getNEP17Balance(h util.Uint160, acc util.Uint160, bw *io.BufBinWriter) (*big.Int, error) {
+	if bw == nil {
+		bw = io.NewBufBinWriter()
+	} else {
+		bw.Reset()
+	}
+	emit.AppCall(bw.BinWriter, h, "balanceOf", callflag.ReadStates, acc)
+	if bw.Err != nil {
+		return nil, fmt.Errorf("failed to create `balanceOf` invocation script: %w", bw.Err)
+	}
+	script := bw.Bytes()
+	tx := &transaction.Transaction{Script: script}
+	v := s.chain.GetTestVM(trigger.Application, tx, nil)
+	v.GasLimit = core.HeaderVerificationGasLimit
+	v.LoadScriptWithFlags(script, callflag.All)
+	err := v.Run()
+	if err != nil {
+		return nil, fmt.Errorf("failed to run `balanceOf` for %s: %w", h.StringLE(), err)
+	}
+	if v.Estack().Len() != 1 {
+		return nil, fmt.Errorf("invalid `balanceOf` return values count: expected 1, got %d", v.Estack().Len())
+	}
+	res, err := v.Estack().Pop().Item().TryInteger()
+	if err != nil {
+		return nil, fmt.Errorf("unexpected `balanceOf` result type: %w", err)
+	}
+	return res, nil
+}
+
 func getTimestampsAndLimit(ps request.Params, index int) (uint64, uint64, int, int, error) {
 	var start, end uint64
 	var limit, page int