Merge pull request #1065 from nspcc-dev/fix/native

core,native: persist native contract via VM

pkg/core/blockchain.go

@@ -363,7 +363,20 @@ func (bc *Blockchain) notificationDispatcher() {
 			// We don't want to waste time looping through transactions when there are no
 			// subscribers.
 			if len(txFeed) != 0 || len(notificationFeed) != 0 || len(executionFeed) != 0 {
-				var aerIdx int
+				aer := event.appExecResults[0]
+				if !aer.TxHash.Equals(event.block.Hash()) {
+					panic("inconsistent application execution results")
+				}
+				for ch := range executionFeed {
+					ch <- aer
+				}
+				for i := range aer.Events {
+					for ch := range notificationFeed {
+						ch <- &aer.Events[i]
+					}
+				}
+
+				aerIdx := 1
 				for _, tx := range event.block.Transactions {
 					aer := event.appExecResults[aerIdx]
 					if !aer.TxHash.Equals(tx.Hash()) {
@@ -547,7 +560,7 @@ func (bc *Blockchain) processHeader(h *block.Header, batch storage.Batch, header
 // and all tests are in place, we can make a more optimized and cleaner implementation.
 func (bc *Blockchain) storeBlock(block *block.Block) error {
 	cache := dao.NewCached(bc.dao)
-	appExecResults := make([]*state.AppExecResult, 0, len(block.Transactions))
+	appExecResults := make([]*state.AppExecResult, 0, 1+len(block.Transactions))
 	if err := cache.StoreAsBlock(block); err != nil {
 		return err
 	}
@@ -556,6 +569,33 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
 		return err
 	}
 
+	if block.Index > 0 {
+		systemInterop := bc.newInteropContext(trigger.System, cache, block, nil)
+		v := SpawnVM(systemInterop)
+		v.LoadScriptWithFlags(bc.contracts.GetPersistScript(), smartcontract.AllowModifyStates|smartcontract.AllowCall)
+		if err := v.Run(); err != nil {
+			return errors.Wrap(err, "can't persist native contracts")
+		} else if _, err := systemInterop.DAO.Persist(); err != nil {
+			return errors.Wrap(err, "can't persist `onPersist` changes")
+		}
+		for i := range systemInterop.Notifications {
+			bc.handleNotification(&systemInterop.Notifications[i], cache, block, block.Hash())
+		}
+		aer := &state.AppExecResult{
+			TxHash:      block.Hash(), // application logs can be retrieved by block hash
+			Trigger:     trigger.System,
+			VMState:     v.State(),
+			GasConsumed: v.GasConsumed(),
+			Stack:       v.Estack().ToContractParameters(),
+			Events:      systemInterop.Notifications,
+		}
+		appExecResults = append(appExecResults, aer)
+		err := cache.PutAppExecResult(aer)
+		if err != nil {
+			return errors.Wrap(err, "failed to Store notifications")
+		}
+	}
+
 	for _, tx := range block.Transactions {
 		if err := cache.StoreAsTransaction(tx, block.Index); err != nil {
 			return err
@@ -575,42 +615,8 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
 			if err != nil {
 				return errors.Wrap(err, "failed to persist invocation results")
 			}
-			for _, note := range systemInterop.Notifications {
+			for i := range systemInterop.Notifications {
-				arr, ok := note.Item.Value().([]stackitem.Item)
+				bc.handleNotification(&systemInterop.Notifications[i], cache, block, tx.Hash())
-				if !ok || len(arr) != 4 {
-					continue
-				}
-				op, ok := arr[0].Value().([]byte)
-				if !ok || (string(op) != "transfer" && string(op) != "Transfer") {
-					continue
-				}
-				var from []byte
-				fromValue := arr[1].Value()
-				// we don't have `from` set when we are minting tokens
-				if fromValue != nil {
-					from, ok = fromValue.([]byte)
-					if !ok {
-						continue
-					}
-				}
-				var to []byte
-				toValue := arr[2].Value()
-				// we don't have `to` set when we are burning tokens
-				if toValue != nil {
-					to, ok = toValue.([]byte)
-					if !ok {
-						continue
-					}
-				}
-				amount, ok := arr[3].Value().(*big.Int)
-				if !ok {
-					bs, ok := arr[3].Value().([]byte)
-					if !ok {
-						continue
-					}
-					amount = bigint.FromBytes(bs)
-				}
-				bc.processNEP5Transfer(cache, tx, block, note.ScriptHash, from, to, amount.Int64())
 			}
 		} else {
 			bc.log.Warn("contract invocation failed",
@@ -633,13 +639,6 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
 		}
 	}
 
-	for i := range bc.contracts.Contracts {
-		systemInterop := bc.newInteropContext(trigger.Application, cache, block, nil)
-		if err := bc.contracts.Contracts[i].OnPersist(systemInterop); err != nil {
-			return err
-		}
-	}
-
 	if bc.config.SaveStorageBatch {
 		bc.lastBatch = cache.DAO.GetBatch()
 	}
@@ -665,6 +664,44 @@ func (bc *Blockchain) storeBlock(block *block.Block) error {
 	return nil
 }
 
+func (bc *Blockchain) handleNotification(note *state.NotificationEvent, d *dao.Cached, b *block.Block, h util.Uint256) {
+	arr, ok := note.Item.Value().([]stackitem.Item)
+	if !ok || len(arr) != 4 {
+		return
+	}
+	op, ok := arr[0].Value().([]byte)
+	if !ok || (string(op) != "transfer" && string(op) != "Transfer") {
+		return
+	}
+	var from []byte
+	fromValue := arr[1].Value()
+	// we don't have `from` set when we are minting tokens
+	if fromValue != nil {
+		from, ok = fromValue.([]byte)
+		if !ok {
+			return
+		}
+	}
+	var to []byte
+	toValue := arr[2].Value()
+	// we don't have `to` set when we are burning tokens
+	if toValue != nil {
+		to, ok = toValue.([]byte)
+		if !ok {
+			return
+		}
+	}
+	amount, ok := arr[3].Value().(*big.Int)
+	if !ok {
+		bs, ok := arr[3].Value().([]byte)
+		if !ok {
+			return
+		}
+		amount = bigint.FromBytes(bs)
+	}
+	bc.processNEP5Transfer(d, h, b, note.ScriptHash, from, to, amount.Int64())
+}
+
 func parseUint160(addr []byte) util.Uint160 {
 	if u, err := util.Uint160DecodeBytesBE(addr); err == nil {
 		return u
@@ -672,7 +709,7 @@ func parseUint160(addr []byte) util.Uint160 {
 	return util.Uint160{}
 }
 
-func (bc *Blockchain) processNEP5Transfer(cache *dao.Cached, tx *transaction.Transaction, b *block.Block, sc util.Uint160, from, to []byte, amount int64) {
+func (bc *Blockchain) processNEP5Transfer(cache *dao.Cached, h util.Uint256, b *block.Block, sc util.Uint160, from, to []byte, amount int64) {
 	toAddr := parseUint160(to)
 	fromAddr := parseUint160(from)
 	transfer := &state.NEP5Transfer{
@@ -681,7 +718,7 @@ func (bc *Blockchain) processNEP5Transfer(cache *dao.Cached, tx *transaction.Tra
 		To:        toAddr,
 		Block:     b.Index,
 		Timestamp: b.Timestamp,
-		Tx:        tx.Hash(),
+		Tx:        h,
 	}
 	if !fromAddr.Equals(util.Uint160{}) {
 		balances, err := cache.GetNEP5Balances(fromAddr)

pkg/core/blockchain_test.go

@@ -257,13 +257,16 @@ func TestSubscriptions(t *testing.T) {
 	require.NoError(t, err)
 	require.Eventually(t, func() bool { return len(blockCh) != 0 }, time.Second, 10*time.Millisecond)
 	assert.Empty(t, notificationCh)
-	assert.Empty(t, executionCh)
+	assert.Len(t, executionCh, 1)
 	assert.Empty(t, txCh)
 
 	b := <-blockCh
 	assert.Equal(t, blocks[0], b)
 	assert.Empty(t, blockCh)
 
+	aer := <-executionCh
+	assert.Equal(t, b.Hash(), aer.TxHash)
+
 	script := io.NewBufBinWriter()
 	emit.Bytes(script.BinWriter, []byte("yay!"))
 	emit.Syscall(script.BinWriter, "System.Runtime.Notify")
@@ -308,6 +311,17 @@ func TestSubscriptions(t *testing.T) {
 	require.Equal(t, invBlock, b)
 	assert.Empty(t, blockCh)
 
+	exec := <-executionCh
+	require.Equal(t, b.Hash(), exec.TxHash)
+	require.Equal(t, exec.VMState, "HALT")
+
+	// 3 burn events for every tx and 1 mint for primary node
+	require.True(t, len(notificationCh) >= 4)
+	for i := 0; i < 4; i++ {
+		notif := <-notificationCh
+		require.Equal(t, bc.contracts.GAS.Hash, notif.ScriptHash)
+	}
+
 	// Follow in-block transaction order.
 	for _, txExpected := range invBlock.Transactions {
 		tx := <-txCh

pkg/core/interop/context.go

@@ -79,7 +79,6 @@ type MethodAndPrice struct {
 type Contract interface {
 	Initialize(*Context) error
 	Metadata() *ContractMD
-	OnPersist(*Context) error
 }
 
 // ContractMD represents native contract instance.

pkg/core/native/contract.go

@@ -4,8 +4,10 @@ import (
 	"fmt"
 
 	"github.com/nspcc-dev/neo-go/pkg/core/interop"
+	"github.com/nspcc-dev/neo-go/pkg/io"
 	"github.com/nspcc-dev/neo-go/pkg/util"
 	"github.com/nspcc-dev/neo-go/pkg/vm"
+	"github.com/nspcc-dev/neo-go/pkg/vm/emit"
 	"github.com/pkg/errors"
 )
 
@@ -14,6 +16,8 @@ type Contracts struct {
 	NEO       *NEO
 	GAS       *GAS
 	Contracts []interop.Contract
+	// persistScript is vm script which executes "onPersist" method of every native contract.
+	persistScript []byte
 }
 
 // ByHash returns native contract with the specified hash.
@@ -53,13 +57,26 @@ func NewContracts() *Contracts {
 	return cs
 }
 
+// GetPersistScript returns VM script calling "onPersist" method of every native contract.
+func (cs *Contracts) GetPersistScript() []byte {
+	if cs.persistScript != nil {
+		return cs.persistScript
+	}
+	w := io.NewBufBinWriter()
+	for i := range cs.Contracts {
+		md := cs.Contracts[i].Metadata()
+		emit.AppCallWithOperationAndArgs(w.BinWriter, md.Hash, "onPersist")
+	}
+	cs.persistScript = w.Bytes()
+	return cs.persistScript
+}
+
 // GetNativeInterop returns an interop getter for a given set of contracts.
 func (cs *Contracts) GetNativeInterop(ic *interop.Context) func(uint32) *vm.InteropFuncPrice {
 	return func(id uint32) *vm.InteropFuncPrice {
 		if c := cs.ByID(id); c != nil {
 			return &vm.InteropFuncPrice{
-				Func:  getNativeInterop(ic, c),
+				Func: getNativeInterop(ic, c),
-				Price: 0, // TODO price func
 			}
 		}
 		return nil
@@ -82,6 +99,9 @@ func getNativeInterop(ic *interop.Context, c interop.Contract) func(v *vm.VM) er
 		if !v.Context().GetCallFlags().Has(m.RequiredFlags) {
 			return errors.New("missing call flags")
 		}
+		if !v.AddGas(util.Fixed8(m.Price)) {
+			return errors.New("gas limit exceeded")
+		}
 		result := m.Func(ic, args)
 		v.Estack().PushVal(result)
 		return nil

pkg/core/native/native_gas.go

@@ -34,12 +34,16 @@ func NewGAS() *GAS {
 	nep5.symbol = "gas"
 	nep5.decimals = 8
 	nep5.factor = GASFactor
-	nep5.onPersist = chainOnPersist(g.onPersist, g.OnPersist)
+	nep5.onPersist = chainOnPersist(nep5.OnPersist, g.OnPersist)
 	nep5.incBalance = g.increaseBalance
 	nep5.ContractID = gasContractID
 
 	g.nep5TokenNative = *nep5
 
+	onp := g.Methods["onPersist"]
+	onp.Func = getOnPersistWrapper(g.onPersist)
+	g.Methods["onPersist"] = onp
+
 	return g
 }
 

pkg/core/native/native_neo.go

@@ -68,39 +68,43 @@ func NewNEO() *NEO {
 	nep5.symbol = "neo"
 	nep5.decimals = 0
 	nep5.factor = 1
-	nep5.onPersist = chainOnPersist(n.onPersist, n.OnPersist)
+	nep5.onPersist = chainOnPersist(nep5.OnPersist, n.OnPersist)
 	nep5.incBalance = n.increaseBalance
 	nep5.ContractID = neoContractID
 
 	n.nep5TokenNative = *nep5
 
+	onp := n.Methods["onPersist"]
+	onp.Func = getOnPersistWrapper(n.onPersist)
+	n.Methods["onPersist"] = onp
+
 	desc := newDescriptor("unclaimedGas", smartcontract.IntegerType,
 		manifest.NewParameter("account", smartcontract.Hash160Type),
 		manifest.NewParameter("end", smartcontract.IntegerType))
-	md := newMethodAndPrice(n.unclaimedGas, 1, smartcontract.AllowStates)
+	md := newMethodAndPrice(n.unclaimedGas, 3000000, smartcontract.AllowStates)
 	n.AddMethod(md, desc, true)
 
 	desc = newDescriptor("registerValidator", smartcontract.BoolType,
 		manifest.NewParameter("pubkey", smartcontract.PublicKeyType))
-	md = newMethodAndPrice(n.registerValidator, 1, smartcontract.AllowModifyStates)
+	md = newMethodAndPrice(n.registerValidator, 5000000, smartcontract.AllowModifyStates)
 	n.AddMethod(md, desc, false)
 
 	desc = newDescriptor("vote", smartcontract.BoolType,
 		manifest.NewParameter("account", smartcontract.Hash160Type),
 		manifest.NewParameter("pubkeys", smartcontract.ArrayType))
-	md = newMethodAndPrice(n.vote, 1, smartcontract.AllowModifyStates)
+	md = newMethodAndPrice(n.vote, 500000000, smartcontract.AllowModifyStates)
 	n.AddMethod(md, desc, false)
 
 	desc = newDescriptor("getRegisteredValidators", smartcontract.ArrayType)
-	md = newMethodAndPrice(n.getRegisteredValidatorsCall, 1, smartcontract.AllowStates)
+	md = newMethodAndPrice(n.getRegisteredValidatorsCall, 100000000, smartcontract.AllowStates)
 	n.AddMethod(md, desc, true)
 
 	desc = newDescriptor("getValidators", smartcontract.ArrayType)
-	md = newMethodAndPrice(n.getValidators, 1, smartcontract.AllowStates)
+	md = newMethodAndPrice(n.getValidators, 100000000, smartcontract.AllowStates)
 	n.AddMethod(md, desc, true)
 
 	desc = newDescriptor("getNextBlockValidators", smartcontract.ArrayType)
-	md = newMethodAndPrice(n.getNextBlockValidators, 1, smartcontract.AllowStates)
+	md = newMethodAndPrice(n.getNextBlockValidators, 100000000, smartcontract.AllowStates)
 	n.AddMethod(md, desc, true)
 
 	return n

pkg/core/native/native_nep5.go

@@ -10,6 +10,7 @@ import (
 	"github.com/nspcc-dev/neo-go/pkg/encoding/bigint"
 	"github.com/nspcc-dev/neo-go/pkg/smartcontract"
 	"github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest"
+	"github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger"
 	"github.com/nspcc-dev/neo-go/pkg/util"
 	"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
 )
@@ -77,6 +78,10 @@ func newNEP5Native(name string) *nep5TokenNative {
 	md = newMethodAndPrice(n.Transfer, 1, smartcontract.AllowModifyStates)
 	n.AddMethod(md, desc, false)
 
+	desc = newDescriptor("onPersist", smartcontract.BoolType)
+	md = newMethodAndPrice(getOnPersistWrapper(n.OnPersist), 0, smartcontract.AllowModifyStates)
+	n.AddMethod(md, desc, false)
+
 	n.AddEvent("Transfer", desc.Parameters...)
 
 	return n
@@ -219,8 +224,7 @@ func (c *nep5TokenNative) burn(ic *interop.Context, h util.Uint160, amount *big.
 	if amount.Sign() == 0 {
 		return
 	}
-	amount = new(big.Int).Neg(amount)
+	c.addTokens(ic, h, new(big.Int).Neg(amount))
-	c.addTokens(ic, h, amount)
 	c.emitTransfer(ic, &h, nil, amount)
 }
 
@@ -250,7 +254,10 @@ func (c *nep5TokenNative) addTokens(ic *interop.Context, h util.Uint160, amount
 }
 
 func (c *nep5TokenNative) OnPersist(ic *interop.Context) error {
-	return c.onPersist(ic)
+	if ic.Trigger != trigger.System {
+		return errors.New("onPersist should be triggerred by system")
+	}
+	return nil
 }
 
 func newDescriptor(name string, ret smartcontract.ParamType, ps ...manifest.Parameter) *manifest.Method {
@@ -311,3 +318,9 @@ func (s nep5ScriptHash) GetEntryScriptHash() util.Uint160 {
 func (s nep5ScriptHash) GetCurrentScriptHash() util.Uint160 {
 	return s.currentScriptHash
 }
+
+func getOnPersistWrapper(f func(ic *interop.Context) error) interop.Method {
+	return func(ic *interop.Context, _ []stackitem.Item) stackitem.Item {
+		return stackitem.NewBool(f(ic) == nil)
+	}
+}

pkg/core/native_contract_test.go

@@ -1,7 +1,6 @@
 package core
 
 import (
-	"errors"
 	"testing"
 
 	"github.com/nspcc-dev/neo-go/pkg/core/interop"
@@ -10,6 +9,7 @@ import (
 	"github.com/nspcc-dev/neo-go/pkg/io"
 	"github.com/nspcc-dev/neo-go/pkg/smartcontract"
 	"github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest"
+	"github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger"
 	"github.com/nspcc-dev/neo-go/pkg/vm/emit"
 	"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
 	"github.com/stretchr/testify/require"
@@ -28,12 +28,15 @@ func (tn *testNative) Metadata() *interop.ContractMD {
 	return &tn.meta
 }
 
-func (tn *testNative) OnPersist(ic *interop.Context) error {
+func (tn *testNative) OnPersist(ic *interop.Context, _ []stackitem.Item) stackitem.Item {
+	if ic.Trigger != trigger.System {
+		panic("invalid trigger")
+	}
 	select {
 	case tn.blocks <- ic.Block.Index:
-		return nil
+		return stackitem.NewBool(true)
 	default:
-		return errors.New("error on persist")
+		return stackitem.NewBool(false)
 	}
 }
 
@@ -44,6 +47,8 @@ func (bc *Blockchain) registerNative(c interop.Contract) {
 	bc.contracts.Contracts = append(bc.contracts.Contracts, c)
 }
 
+const testSumPrice = 1000000
+
 func newTestNative() *testNative {
 	tn := &testNative{
 		meta:   *interop.NewContractMD("Test.Native.Sum"),
@@ -59,11 +64,15 @@ func newTestNative() *testNative {
 	}
 	md := &interop.MethodAndPrice{
 		Func:          tn.sum,
-		Price:         1,
+		Price:         testSumPrice,
 		RequiredFlags: smartcontract.NoneFlag,
 	}
 	tn.meta.AddMethod(md, desc, true)
 
+	desc = &manifest.Method{Name: "onPersist", ReturnType: smartcontract.BoolType}
+	md = &interop.MethodAndPrice{Func: tn.OnPersist, RequiredFlags: smartcontract.AllowModifyStates}
+	tn.meta.AddMethod(md, desc, false)
+
 	return tn
 }
 
@@ -92,7 +101,8 @@ func TestNativeContract_Invoke(t *testing.T) {
 	w := io.NewBufBinWriter()
 	emit.AppCallWithOperationAndArgs(w.BinWriter, tn.Metadata().Hash, "sum", int64(14), int64(28))
 	script := w.Bytes()
-	tx := transaction.New(chain.GetConfig().Magic, script, 0)
+	// System.Contract.Call + "sum" itself + opcodes for pushing arguments (PACK is 7000)
+	tx := transaction.New(chain.GetConfig().Magic, script, testSumPrice*2+10000)
 	validUntil := chain.blockHeight + 1
 	tx.ValidUntilBlock = validUntil
 	require.NoError(t, addSender(tx))

pkg/rpc/server/server.go

@@ -578,7 +578,7 @@ func (s *Server) getNEP5Transfers(ps request.Params) (interface{}, *response.Err
 		}
 
 		transfer.Amount = amountToString(-tr.Amount, d)
-		if !tr.From.Equals(util.Uint160{}) {
+		if !tr.To.Equals(util.Uint160{}) {
 			transfer.Address = address.Uint160ToString(tr.To)
 		}
 		bs.Sent = append(bs.Sent, transfer)

pkg/rpc/server/server_test.go

@@ -148,8 +148,8 @@ var rpcTestCases = map[string][]rpcTestCase{
 						},
 						{
 							Asset:       e.chain.UtilityTokenHash(),
-							Amount:      "1023.99976000",
+							Amount:      "923.96934740",
-							LastUpdated: 4,
+							LastUpdated: 6,
 						}},
 					Address: testchain.PrivateKeyByID(0).GetScriptHash().StringLE(),
 				}
@@ -256,6 +256,27 @@ var rpcTestCases = map[string][]rpcTestCase{
 					},
 					Address: testchain.PrivateKeyByID(0).Address(),
 				}
+
+				// take burned gas into account
+				u := testchain.PrivateKeyByID(0).GetScriptHash()
+				for i := 0; i <= int(e.chain.BlockHeight()); i++ {
+					h := e.chain.GetHeaderHash(i)
+					b, err := e.chain.GetBlock(h)
+					require.NoError(t, err)
+					for j := range b.Transactions {
+						if u.Equals(b.Transactions[j].Sender) {
+							amount := b.Transactions[j].SystemFee + b.Transactions[j].NetworkFee
+							expected.Sent = append(expected.Sent, result.NEP5Transfer{
+								Timestamp: b.Timestamp,
+								Asset:     e.chain.UtilityTokenHash(),
+								Address:   "", // burn has empty receiver
+								Amount:    amountToString(int64(amount), 8),
+								Index:     b.Index,
+								TxHash:    b.Hash(),
+							})
+						}
+					}
+				}
 				require.Equal(t, expected.Address, res.Address)
 				require.ElementsMatch(t, expected.Sent, res.Sent)
 				require.ElementsMatch(t, expected.Received, res.Received)

pkg/rpc/server/subscription_test.go

@@ -97,8 +97,18 @@ func TestSubscriptions(t *testing.T) {
 
 	for _, b := range getTestBlocks(t) {
 		require.NoError(t, chain.AddBlock(b))
-		for range b.Transactions {
+		resp := getNotification(t, respMsgs)
+		require.Equal(t, response.ExecutionEventID, resp.Event)
+		for {
 			resp := getNotification(t, respMsgs)
+			if resp.Event != response.NotificationEventID {
+				break
+			}
+		}
+		for i := 0; i < len(b.Transactions); i++ {
+			if i > 0 {
+				resp = getNotification(t, respMsgs)
+			}
 			require.Equal(t, response.ExecutionEventID, resp.Event)
 			for {
 				resp := getNotification(t, respMsgs)
@@ -109,7 +119,7 @@ func TestSubscriptions(t *testing.T) {
 				break
 			}
 		}
-		resp := getNotification(t, respMsgs)
+		resp = getNotification(t, respMsgs)
 		require.Equal(t, response.BlockEventID, resp.Event)
 	}