Merge pull request #3569 from ixje/applog-invocations

*: add invocations to applicationlog

docs/node-configuration.md

@@ -35,6 +35,7 @@ node-related settings described in the table below.
 | SaveStorageBatch | `bool` | `false` | Enables storage batch saving before every persist. It is similar to StorageDump plugin for C# node. |
 | SkipBlockVerification | `bool` | `false` | Allows to disable verification of received/processed blocks (including cryptographic checks). |
 | StateRoot | [State Root Configuration](#State-Root-Configuration) |  | State root module configuration. See the [State Root Configuration](#State-Root-Configuration) section for details. |
+| SaveInvocations | `bool` | `false` | Determines if additional smart contract invocation details are stored. If enabled, the `getapplicationlog` RPC method will return a new field with invocation details for the transaction. See the [RPC](rpc.md#applicationlog-invocations) documentation for more information. |
 
 ### P2P Configuration
 
@@ -495,6 +496,7 @@ affect this:
 - `GarbageCollectionPeriod` must be the same
 - `KeepOnlyLatestState` must be the same
 - `RemoveUntraceableBlocks` must be the same
+- `SaveInvocations` must be the same
 
 BotlDB is also known to be incompatible between machines with different
 endianness. Nothing is known for LevelDB wrt this, so it's not recommended

docs/rpc.md

@@ -356,6 +356,62 @@ to various blockchain events (with simple event filtering) and receive them on
 the client as JSON-RPC notifications. More details on that are written in the
 [notifications specification](notifications.md).
 
+#### `applicationlog` call invocations
+
+The `SaveInvocations` configuration setting causes the RPC server to store smart contract
+invocation details as part of the application logs. This feature is specifically useful to
+capture information in the absence of `System.Runtime.Notify` calls for the given smart 
+contract method. Other use-cases are described in [this issue](https://github.com/neo-project/neo/issues/3386).
+
+Example transaction on Testnet which interacts with the native PolicyContract:
+```json
+{
+  "id": 1,
+  "jsonrpc": "2.0",
+  "result": {
+    "txid": "0xd6fe5f61d9cb34d6324db1be42c056d02ba1f1f6cd0bd3f3c6bb24faaaeef2a9",
+    "executions": [
+      {
+        "trigger": "Application",
+        "vmstate": "HALT",
+        "gasconsumed": "2028120",
+        "stack": [
+          {
+            "type": "Any"
+          }
+        ],
+        "notifications": [],
+        "exception": null,
+        "invocations": [
+          {
+            "hash": "0xcc5e4edd9f5f8dba8bb65734541df7a1c081c67b",
+            "method": "setFeePerByte",
+            "arguments": {
+              "type": "Array",
+              "value": [
+                {
+                  "type": "Integer",
+                  "value": "100"
+                }
+              ]
+            },
+            "argumentscount": 1,
+            "truncated": false
+          }
+        ]
+      }
+    ]
+  }
+}
+```
+
+For security reasons the `arguments` field data may result in `null` if the count exceeds 2048. 
+In such case the `Truncated` field will be set to `true`.
+
+The invocation records are presented in a flat structure in the order as how they were executed.
+Note that invocation records for faulted transactions are kept and are present in the 
+applicationlog. This behaviour differs from notifications which are omitted for faulted transactions.
+
 ## Reference
 
 * [JSON-RPC 2.0 Specification](http://www.jsonrpc.org/specification)

pkg/config/ledger_config.go

@@ -22,6 +22,8 @@ type Ledger struct {
 	// SkipBlockVerification allows to disable verification of received
 	// blocks (including cryptographic checks).
 	SkipBlockVerification bool `yaml:"SkipBlockVerification"`
+	// SaveInvocations enables smart contract invocation data saving.
+	SaveInvocations bool `yaml:"SaveInvocations"`
 }
 
 // Blockchain is a set of settings for core.Blockchain to use, it includes protocol

pkg/core/blockchain.go

@@ -451,6 +451,7 @@ func (bc *Blockchain) init() error {
 			KeepOnlyLatestState:        bc.config.Ledger.KeepOnlyLatestState,
 			Magic:                      uint32(bc.config.Magic),
 			Value:                      version,
+			SaveInvocations:            bc.config.SaveInvocations,
 		}
 		bc.dao.PutVersion(ver)
 		bc.dao.Version = ver
@@ -488,6 +489,10 @@ func (bc *Blockchain) init() error {
 		return fmt.Errorf("protocol configuration Magic mismatch (old=%v, new=%v)",
 			ver.Magic, bc.config.Magic)
 	}
+	if ver.SaveInvocations != bc.config.SaveInvocations {
+		return fmt.Errorf("SaveInvocations setting mismatch (old=%v, new=%v)",
+			ver.SaveInvocations, bc.config.SaveInvocations)
+	}
 	bc.dao.Version = ver
 	bc.persistent.Version = ver
 
@@ -1754,6 +1759,7 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error
 				Stack:          v.Estack().ToArray(),
 				Events:         systemInterop.Notifications,
 				FaultException: faultException,
+				Invocations:    systemInterop.InvocationCalls,
 			},
 		}
 		appExecResults = append(appExecResults, aer)

pkg/core/dao/dao.go

@@ -448,6 +448,7 @@ type Version struct {
 	KeepOnlyLatestState        bool
 	Magic                      uint32
 	Value                      string
+	SaveInvocations            bool
 }
 
 const (
@@ -455,6 +456,7 @@ const (
 	p2pSigExtensionsBit
 	p2pStateExchangeExtensionsBit
 	keepOnlyLatestStateBit
+	saveInvocationsBit
 )
 
 // FromBytes decodes v from a byte-slice.
@@ -482,6 +484,7 @@ func (v *Version) FromBytes(data []byte) error {
 	v.P2PSigExtensions = data[i+2]&p2pSigExtensionsBit != 0
 	v.P2PStateExchangeExtensions = data[i+2]&p2pStateExchangeExtensionsBit != 0
 	v.KeepOnlyLatestState = data[i+2]&keepOnlyLatestStateBit != 0
+	v.SaveInvocations = data[i+2]&saveInvocationsBit != 0
 
 	m := i + 3
 	if len(data) == m+4 {
@@ -505,6 +508,9 @@ func (v *Version) Bytes() []byte {
 	if v.KeepOnlyLatestState {
 		mask |= keepOnlyLatestStateBit
 	}
+	if v.SaveInvocations {
+		mask |= saveInvocationsBit
+	}
 	res := append([]byte(v.Value), '\x00', byte(v.StoragePrefix), mask)
 	res = binary.LittleEndian.AppendUint32(res, v.Magic)
 	return res

pkg/core/interop/context.go

@@ -63,6 +63,7 @@ type Context struct {
 	VM               *vm.VM
 	Functions        []Function
 	Invocations      map[util.Uint160]int
+	InvocationCalls  []state.ContractInvocation
 	cancelFuncs      []context.CancelFunc
 	getContract      func(*dao.Simple, util.Uint160) (*state.Contract, error)
 	baseExecFee      int64
@@ -70,6 +71,7 @@ type Context struct {
 	loadToken        func(ic *Context, id int32) error
 	GetRandomCounter uint32
 	signers          []transaction.Signer
+	SaveInvocations  bool
 }
 
 // NewContext returns new interop context.
@@ -78,22 +80,23 @@ func NewContext(trigger trigger.Type, bc Ledger, d *dao.Simple, baseExecFee, bas
 	loadTokenFunc func(ic *Context, id int32) error,
 	block *block.Block, tx *transaction.Transaction, log *zap.Logger) *Context {
 	dao := d.GetPrivate()
-	cfg := bc.GetConfig().ProtocolConfiguration
+	cfg := bc.GetConfig()
 	return &Context{
-		Chain:          bc,
-		Network:        uint32(cfg.Magic),
-		Hardforks:      cfg.Hardforks,
-		Natives:        natives,
-		Trigger:        trigger,
-		Block:          block,
-		Tx:             tx,
-		DAO:            dao,
-		Log:            log,
-		Invocations:    make(map[util.Uint160]int),
-		getContract:    getContract,
-		baseExecFee:    baseExecFee,
-		baseStorageFee: baseStorageFee,
-		loadToken:      loadTokenFunc,
+		Chain:           bc,
+		Network:         uint32(cfg.Magic),
+		Hardforks:       cfg.Hardforks,
+		Natives:         natives,
+		Trigger:         trigger,
+		Block:           block,
+		Tx:              tx,
+		DAO:             dao,
+		Log:             log,
+		Invocations:     make(map[util.Uint160]int),
+		getContract:     getContract,
+		baseExecFee:     baseExecFee,
+		baseStorageFee:  baseStorageFee,
+		loadToken:       loadTokenFunc,
+		SaveInvocations: cfg.SaveInvocations,
 	}
 }
 

pkg/core/interop/contract/call.go

@@ -1,6 +1,7 @@
 package contract
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"math/big"
@@ -69,6 +70,18 @@ func Call(ic *interop.Context) error {
 		return fmt.Errorf("method not found: %s/%d", method, len(args))
 	}
 	hasReturn := md.ReturnType != smartcontract.VoidType
+
+	if ic.SaveInvocations {
+		var (
+			arrCount = len(args)
+			argBytes []byte
+		)
+		if argBytes, err = ic.DAO.GetItemCtx().Serialize(stackitem.NewArray(args), false); err != nil {
+			argBytes = nil
+		}
+		ci := state.NewContractInvocation(u, method, bytes.Clone(argBytes), uint32(arrCount))
+		ic.InvocationCalls = append(ic.InvocationCalls, *ci)
+	}
 	return callInternal(ic, cs, method, fs, hasReturn, args, true)
 }
 

pkg/core/state/contract_invocation.go

@@ -0,0 +1,120 @@
+package state
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/nspcc-dev/neo-go/pkg/io"
+	"github.com/nspcc-dev/neo-go/pkg/util"
+	"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
+)
+
+// ContractInvocation contains method call information.
+// The Arguments field will be nil if serialization of the arguments exceeds the predefined limit
+// of [stackitem.MaxSerialized] (for security reasons). In that case Truncated will be set to true.
+type ContractInvocation struct {
+	Hash   util.Uint160 `json:"contract"`
+	Method string       `json:"method"`
+	// Arguments are the arguments as passed to the `args` parameter of System.Contract.Call
+	// for use in the RPC Server and RPC Client.
+	Arguments *stackitem.Array `json:"arguments"`
+	// argumentsBytes is the serialized arguments used at the interop level.
+	argumentsBytes []byte
+	ArgumentsCount uint32 `json:"argumentscount"`
+	Truncated      bool   `json:"truncated"`
+}
+
+// contractInvocationAux is an auxiliary struct for ContractInvocation JSON marshalling.
+type contractInvocationAux struct {
+	Hash           util.Uint160    `json:"hash"`
+	Method         string          `json:"method"`
+	Arguments      json.RawMessage `json:"arguments,omitempty"`
+	ArgumentsCount uint32          `json:"argumentscount"`
+	Truncated      bool            `json:"truncated"`
+}
+
+// NewContractInvocation returns a new ContractInvocation.
+func NewContractInvocation(hash util.Uint160, method string, argBytes []byte, argCnt uint32) *ContractInvocation {
+	return &ContractInvocation{
+		Hash:           hash,
+		Method:         method,
+		argumentsBytes: argBytes,
+		ArgumentsCount: argCnt,
+		Truncated:      argBytes == nil,
+	}
+}
+
+// DecodeBinary implements the Serializable interface.
+func (ci *ContractInvocation) DecodeBinary(r *io.BinReader) {
+	ci.Hash.DecodeBinary(r)
+	ci.Method = r.ReadString()
+	ci.ArgumentsCount = r.ReadU32LE()
+	ci.Truncated = r.ReadBool()
+	if !ci.Truncated {
+		ci.argumentsBytes = r.ReadVarBytes()
+	}
+}
+
+// EncodeBinary implements the Serializable interface.
+func (ci *ContractInvocation) EncodeBinary(w *io.BinWriter) {
+	ci.EncodeBinaryWithContext(w, stackitem.NewSerializationContext())
+}
+
+// EncodeBinaryWithContext is the same as EncodeBinary, but allows to efficiently reuse
+// stack item serialization context.
+func (ci *ContractInvocation) EncodeBinaryWithContext(w *io.BinWriter, sc *stackitem.SerializationContext) {
+	ci.Hash.EncodeBinary(w)
+	w.WriteString(ci.Method)
+	w.WriteU32LE(ci.ArgumentsCount)
+	w.WriteBool(ci.Truncated)
+	if !ci.Truncated {
+		w.WriteVarBytes(ci.argumentsBytes)
+	}
+}
+
+// MarshalJSON implements the json.Marshaler interface.
+func (ci ContractInvocation) MarshalJSON() ([]byte, error) {
+	var item []byte
+	if ci.Arguments == nil && ci.argumentsBytes != nil {
+		si, err := stackitem.Deserialize(ci.argumentsBytes)
+		if err != nil {
+			return nil, err
+		}
+		item, err = stackitem.ToJSONWithTypes(si.(*stackitem.Array))
+		if err != nil {
+			item = nil
+		}
+	}
+	return json.Marshal(contractInvocationAux{
+		Hash:           ci.Hash,
+		Method:         ci.Method,
+		Arguments:      item,
+		ArgumentsCount: ci.ArgumentsCount,
+		Truncated:      ci.Truncated,
+	})
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface.
+func (ci *ContractInvocation) UnmarshalJSON(data []byte) error {
+	aux := new(contractInvocationAux)
+	if err := json.Unmarshal(data, aux); err != nil {
+		return err
+	}
+	var args *stackitem.Array
+	if aux.Arguments != nil {
+		arguments, err := stackitem.FromJSONWithTypes(aux.Arguments)
+		if err != nil {
+			return err
+		}
+		if t := arguments.Type(); t != stackitem.ArrayT {
+			return fmt.Errorf("failed to convert invocation state of type %s to array", t.String())
+		}
+		args = arguments.(*stackitem.Array)
+	}
+	ci.Method = aux.Method
+	ci.Hash = aux.Hash
+	ci.ArgumentsCount = aux.ArgumentsCount
+	ci.Truncated = aux.Truncated
+	ci.Arguments = args
+	return nil
+}

pkg/core/state/contract_invocation_test.go

@@ -0,0 +1,51 @@
+package state
+
+import (
+	"testing"
+
+	json "github.com/nspcc-dev/go-ordered-json"
+	"github.com/nspcc-dev/neo-go/internal/testserdes"
+	"github.com/nspcc-dev/neo-go/pkg/util"
+	"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
+	"github.com/stretchr/testify/require"
+)
+
+func TestContractInvocation_MarshalUnmarshalJSON(t *testing.T) {
+	t.Run("truncated", func(t *testing.T) {
+		ci := NewContractInvocation(util.Uint160{}, "fakeMethodCall", nil, 1)
+		testserdes.MarshalUnmarshalJSON(t, ci, new(ContractInvocation))
+	})
+	t.Run("not truncated", func(t *testing.T) {
+		si := stackitem.NewArray([]stackitem.Item{stackitem.NewBool(false)})
+		argBytes, err := stackitem.NewSerializationContext().Serialize(si, false)
+		require.NoError(t, err)
+
+		ci := NewContractInvocation(util.Uint160{}, "fakeMethodCall", argBytes, 1)
+		// Marshal and Unmarshal are asymmetric, test manually
+		out, err := json.Marshal(&ci)
+		require.NoError(t, err)
+		var ci2 ContractInvocation
+		err = json.Unmarshal(out, &ci2)
+		require.NoError(t, err)
+		require.Equal(t, ci.Hash, ci2.Hash)
+		require.Equal(t, ci.Method, ci2.Method)
+		require.Equal(t, ci.Truncated, ci2.Truncated)
+		require.Equal(t, ci.ArgumentsCount, ci2.ArgumentsCount)
+		require.Equal(t, si, ci2.Arguments)
+	})
+}
+
+func TestContractInvocation_EncodeDecodeBinary(t *testing.T) {
+	t.Run("truncated", func(t *testing.T) {
+		ci := NewContractInvocation(util.Uint160{}, "fakeMethodCall", nil, 1)
+		testserdes.EncodeDecodeBinary(t, ci, new(ContractInvocation))
+	})
+	t.Run("not truncated", func(t *testing.T) {
+		si := stackitem.NewArray([]stackitem.Item{stackitem.NewBool(false)})
+		argBytes, err := stackitem.NewSerializationContext().Serialize(si, false)
+		require.NoError(t, err)
+
+		ci := NewContractInvocation(util.Uint160{}, "fakeMethodCall", argBytes, 1)
+		testserdes.EncodeDecodeBinary(t, ci, new(ContractInvocation))
+	})
+}

pkg/core/state/notification_event.go

@@ -12,6 +12,18 @@ import (
 	"github.com/nspcc-dev/neo-go/pkg/vm/vmstate"
 )
 
+const (
+	// saveInvocationsBit acts as a marker using the VMState to indicate whether contract
+	// invocations (tracked by the VM) have been stored into the DB and thus whether they
+	// should be deserialized upon retrieval. This approach saves 1 byte for all
+	// applicationlogs over	using WriteVarBytes. The original discussion can be found here
+	// https://github.com/nspcc-dev/neo-go/pull/3569#discussion_r1909357541
+	saveInvocationsBit = 0x80
+	// cleanSaveInvocationsBitMask is used to remove the save invocations marker bit from
+	// the VMState.
+	cleanSaveInvocationsBitMask = saveInvocationsBit ^ 0xFF
+)
+
 // NotificationEvent is a tuple of the scripthash that has emitted the Item as a
 // notification and the item itself.
 type NotificationEvent struct {
@@ -78,6 +90,10 @@ func (aer *AppExecResult) EncodeBinary(w *io.BinWriter) {
 func (aer *AppExecResult) EncodeBinaryWithContext(w *io.BinWriter, sc *stackitem.SerializationContext) {
 	w.WriteBytes(aer.Container[:])
 	w.WriteB(byte(aer.Trigger))
+	invocLen := len(aer.Invocations)
+	if invocLen > 0 {
+		aer.VMState |= saveInvocationsBit
+	}
 	w.WriteB(byte(aer.VMState))
 	w.WriteU64LE(uint64(aer.GasConsumed))
 	// Stack items are expected to be marshaled one by one.
@@ -95,6 +111,12 @@ func (aer *AppExecResult) EncodeBinaryWithContext(w *io.BinWriter, sc *stackitem
 		aer.Events[i].EncodeBinaryWithContext(w, sc)
 	}
 	w.WriteVarBytes([]byte(aer.FaultException))
+	if invocLen > 0 {
+		w.WriteVarUint(uint64(invocLen))
+		for i := range aer.Invocations {
+			aer.Invocations[i].EncodeBinaryWithContext(w, sc)
+		}
+	}
 }
 
 // DecodeBinary implements the Serializable interface.
@@ -120,6 +142,10 @@ func (aer *AppExecResult) DecodeBinary(r *io.BinReader) {
 	aer.Stack = arr
 	r.ReadArray(&aer.Events)
 	aer.FaultException = r.ReadString()
+	if aer.VMState&saveInvocationsBit != 0 {
+		r.ReadArray(&aer.Invocations)
+		aer.VMState &= cleanSaveInvocationsBitMask
+	}
 }
 
 // notificationEventAux is an auxiliary struct for NotificationEvent JSON marshalling.
@@ -209,16 +235,18 @@ type Execution struct {
 	Stack          []stackitem.Item
 	Events         []NotificationEvent
 	FaultException string
+	Invocations    []ContractInvocation
 }
 
 // executionAux represents an auxiliary struct for Execution JSON marshalling.
 type executionAux struct {
-	Trigger        string              `json:"trigger"`
-	VMState        string              `json:"vmstate"`
-	GasConsumed    int64               `json:"gasconsumed,string"`
-	Stack          json.RawMessage     `json:"stack"`
-	Events         []NotificationEvent `json:"notifications"`
-	FaultException *string             `json:"exception"`
+	Trigger        string               `json:"trigger"`
+	VMState        string               `json:"vmstate"`
+	GasConsumed    int64                `json:"gasconsumed,string"`
+	Stack          json.RawMessage      `json:"stack"`
+	Events         []NotificationEvent  `json:"notifications"`
+	FaultException *string              `json:"exception"`
+	Invocations    []ContractInvocation `json:"invocations"`
 }
 
 // MarshalJSON implements the json.Marshaler interface.
@@ -246,6 +274,7 @@ func (e Execution) MarshalJSON() ([]byte, error) {
 		Stack:          st,
 		Events:         e.Events,
 		FaultException: exception,
+		Invocations:    e.Invocations,
 	})
 }
 
@@ -287,6 +316,7 @@ func (e *Execution) UnmarshalJSON(data []byte) error {
 	if aux.FaultException != nil {
 		e.FaultException = *aux.FaultException
 	}
+	e.Invocations = aux.Invocations
 	return nil
 }