From 4b2ee9a4240b1388e253b6cad27da2111938358f Mon Sep 17 00:00:00 2001 From: ixje Date: Fri, 24 Jan 2025 15:44:10 +0100 Subject: [PATCH] state, rpc: add invocations to applicationlog Add setting to include smart contract invocations data to the applicationlog. Original issue at https://github.com/neo-project/neo/issues/3386 Signed-off-by: ixje --- docs/node-configuration.md | 2 + docs/rpc.md | 56 ++++++++++ pkg/config/ledger_config.go | 2 + pkg/core/blockchain.go | 6 ++ pkg/core/dao/dao.go | 6 ++ pkg/core/interop/context.go | 33 +++--- pkg/core/interop/contract/call.go | 13 +++ pkg/core/state/contract_invocation.go | 120 +++++++++++++++++++++ pkg/core/state/contract_invocation_test.go | 51 +++++++++ pkg/core/state/notification_event.go | 42 ++++++-- 10 files changed, 310 insertions(+), 21 deletions(-) create mode 100644 pkg/core/state/contract_invocation.go create mode 100644 pkg/core/state/contract_invocation_test.go diff --git a/docs/node-configuration.md b/docs/node-configuration.md index 162ef7909..196d660ca 100644 --- a/docs/node-configuration.md +++ b/docs/node-configuration.md @@ -34,6 +34,7 @@ node-related settings described in the table below. | SaveStorageBatch | `bool` | `false` | Enables storage batch saving before every persist. It is similar to StorageDump plugin for C# node. | | SkipBlockVerification | `bool` | `false` | Allows to disable verification of received/processed blocks (including cryptographic checks). | | StateRoot | [State Root Configuration](#State-Root-Configuration) | | State root module configuration. See the [State Root Configuration](#State-Root-Configuration) section for details. | +| SaveInvocations | `bool` | `false` | Determines if additional smart contract invocation details are stored. If enabled, the `getapplicationlog` RPC method will return a new field with invocation details for the transaction. See the [RPC](rpc.md#applicationlog-invocations) documentation for more information. | ### P2P Configuration @@ -471,6 +472,7 @@ affect this: - `GarbageCollectionPeriod` must be the same - `KeepOnlyLatestState` must be the same - `RemoveUntraceableBlocks` must be the same +- `SaveInvocations` must be the same BotlDB is also known to be incompatible between machines with different endianness. Nothing is known for LevelDB wrt this, so it's not recommended diff --git a/docs/rpc.md b/docs/rpc.md index d450c1b7b..0c5f8c987 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -356,6 +356,62 @@ to various blockchain events (with simple event filtering) and receive them on the client as JSON-RPC notifications. More details on that are written in the [notifications specification](notifications.md). +#### `applicationlog` call invocations + +The `SaveInvocations` configuration setting causes the RPC server to store smart contract +invocation details as part of the application logs. This feature is specifically useful to +capture information in the absence of `System.Runtime.Notify` calls for the given smart +contract method. Other use-cases are described in [this issue](https://github.com/neo-project/neo/issues/3386). + +Example transaction on Testnet which interacts with the native PolicyContract: +```json +{ + "id": 1, + "jsonrpc": "2.0", + "result": { + "txid": "0xd6fe5f61d9cb34d6324db1be42c056d02ba1f1f6cd0bd3f3c6bb24faaaeef2a9", + "executions": [ + { + "trigger": "Application", + "vmstate": "HALT", + "gasconsumed": "2028120", + "stack": [ + { + "type": "Any" + } + ], + "notifications": [], + "exception": null, + "invocations": [ + { + "hash": "0xcc5e4edd9f5f8dba8bb65734541df7a1c081c67b", + "method": "setFeePerByte", + "arguments": { + "type": "Array", + "value": [ + { + "type": "Integer", + "value": "100" + } + ] + }, + "argumentscount": 1, + "truncated": false + } + ] + } + ] + } +} +``` + +For security reasons the `arguments` field data may result in `null` if the count exceeds 2048. +In such case the `Truncated` field will be set to `true`. + +The invocation records are presented in a flat structure in the order as how they were executed. +Note that invocation records for faulted transactions are kept and are present in the +applicationlog. This behaviour differs from notifications which are omitted for faulted transactions. + ## Reference * [JSON-RPC 2.0 Specification](http://www.jsonrpc.org/specification) diff --git a/pkg/config/ledger_config.go b/pkg/config/ledger_config.go index 529a0d017..243f5de5b 100644 --- a/pkg/config/ledger_config.go +++ b/pkg/config/ledger_config.go @@ -19,6 +19,8 @@ type Ledger struct { // SkipBlockVerification allows to disable verification of received // blocks (including cryptographic checks). SkipBlockVerification bool `yaml:"SkipBlockVerification"` + // SaveInvocations enables smart contract invocation data saving. + SaveInvocations bool `yaml:"SaveInvocations"` } // Blockchain is a set of settings for core.Blockchain to use, it includes protocol diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index fcfc1eec0..edd383355 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -417,6 +417,7 @@ func (bc *Blockchain) init() error { KeepOnlyLatestState: bc.config.Ledger.KeepOnlyLatestState, Magic: uint32(bc.config.Magic), Value: version, + SaveInvocations: bc.config.SaveInvocations, } bc.dao.PutVersion(ver) bc.dao.Version = ver @@ -454,6 +455,10 @@ func (bc *Blockchain) init() error { return fmt.Errorf("protocol configuration Magic mismatch (old=%v, new=%v)", ver.Magic, bc.config.Magic) } + if ver.SaveInvocations != bc.config.SaveInvocations { + return fmt.Errorf("SaveInvocations setting mismatch (old=%v, new=%v)", + ver.SaveInvocations, bc.config.SaveInvocations) + } bc.dao.Version = ver bc.persistent.Version = ver @@ -1717,6 +1722,7 @@ func (bc *Blockchain) storeBlock(block *block.Block, txpool *mempool.Pool) error Stack: v.Estack().ToArray(), Events: systemInterop.Notifications, FaultException: faultException, + Invocations: systemInterop.InvocationCalls, }, } appExecResults = append(appExecResults, aer) diff --git a/pkg/core/dao/dao.go b/pkg/core/dao/dao.go index 579d35548..0aa17d7d6 100644 --- a/pkg/core/dao/dao.go +++ b/pkg/core/dao/dao.go @@ -448,6 +448,7 @@ type Version struct { KeepOnlyLatestState bool Magic uint32 Value string + SaveInvocations bool } const ( @@ -455,6 +456,7 @@ const ( p2pSigExtensionsBit p2pStateExchangeExtensionsBit keepOnlyLatestStateBit + saveInvocationsBit ) // FromBytes decodes v from a byte-slice. @@ -482,6 +484,7 @@ func (v *Version) FromBytes(data []byte) error { v.P2PSigExtensions = data[i+2]&p2pSigExtensionsBit != 0 v.P2PStateExchangeExtensions = data[i+2]&p2pStateExchangeExtensionsBit != 0 v.KeepOnlyLatestState = data[i+2]&keepOnlyLatestStateBit != 0 + v.SaveInvocations = data[i+2]&saveInvocationsBit != 0 m := i + 3 if len(data) == m+4 { @@ -505,6 +508,9 @@ func (v *Version) Bytes() []byte { if v.KeepOnlyLatestState { mask |= keepOnlyLatestStateBit } + if v.SaveInvocations { + mask |= saveInvocationsBit + } res := append([]byte(v.Value), '\x00', byte(v.StoragePrefix), mask) res = binary.LittleEndian.AppendUint32(res, v.Magic) return res diff --git a/pkg/core/interop/context.go b/pkg/core/interop/context.go index d9ff39426..6cbd6d4ac 100644 --- a/pkg/core/interop/context.go +++ b/pkg/core/interop/context.go @@ -63,6 +63,7 @@ type Context struct { VM *vm.VM Functions []Function Invocations map[util.Uint160]int + InvocationCalls []state.ContractInvocation cancelFuncs []context.CancelFunc getContract func(*dao.Simple, util.Uint160) (*state.Contract, error) baseExecFee int64 @@ -70,6 +71,7 @@ type Context struct { loadToken func(ic *Context, id int32) error GetRandomCounter uint32 signers []transaction.Signer + SaveInvocations bool } // NewContext returns new interop context. @@ -78,22 +80,23 @@ func NewContext(trigger trigger.Type, bc Ledger, d *dao.Simple, baseExecFee, bas loadTokenFunc func(ic *Context, id int32) error, block *block.Block, tx *transaction.Transaction, log *zap.Logger) *Context { dao := d.GetPrivate() - cfg := bc.GetConfig().ProtocolConfiguration + cfg := bc.GetConfig() return &Context{ - Chain: bc, - Network: uint32(cfg.Magic), - Hardforks: cfg.Hardforks, - Natives: natives, - Trigger: trigger, - Block: block, - Tx: tx, - DAO: dao, - Log: log, - Invocations: make(map[util.Uint160]int), - getContract: getContract, - baseExecFee: baseExecFee, - baseStorageFee: baseStorageFee, - loadToken: loadTokenFunc, + Chain: bc, + Network: uint32(cfg.Magic), + Hardforks: cfg.Hardforks, + Natives: natives, + Trigger: trigger, + Block: block, + Tx: tx, + DAO: dao, + Log: log, + Invocations: make(map[util.Uint160]int), + getContract: getContract, + baseExecFee: baseExecFee, + baseStorageFee: baseStorageFee, + loadToken: loadTokenFunc, + SaveInvocations: cfg.SaveInvocations, } } diff --git a/pkg/core/interop/contract/call.go b/pkg/core/interop/contract/call.go index 60f8ac9c5..283e89da7 100644 --- a/pkg/core/interop/contract/call.go +++ b/pkg/core/interop/contract/call.go @@ -1,6 +1,7 @@ package contract import ( + "bytes" "errors" "fmt" "math/big" @@ -69,6 +70,18 @@ func Call(ic *interop.Context) error { return fmt.Errorf("method not found: %s/%d", method, len(args)) } hasReturn := md.ReturnType != smartcontract.VoidType + + if ic.SaveInvocations { + var ( + arrCount = len(args) + argBytes []byte + ) + if argBytes, err = ic.DAO.GetItemCtx().Serialize(stackitem.NewArray(args), false); err != nil { + argBytes = nil + } + ci := state.NewContractInvocation(u, method, bytes.Clone(argBytes), uint32(arrCount)) + ic.InvocationCalls = append(ic.InvocationCalls, *ci) + } return callInternal(ic, cs, method, fs, hasReturn, args, true) } diff --git a/pkg/core/state/contract_invocation.go b/pkg/core/state/contract_invocation.go new file mode 100644 index 000000000..8e6111632 --- /dev/null +++ b/pkg/core/state/contract_invocation.go @@ -0,0 +1,120 @@ +package state + +import ( + "encoding/json" + "fmt" + + "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" +) + +// ContractInvocation contains method call information. +// The Arguments field will be nil if serialization of the arguments exceeds the predefined limit +// of [stackitem.MaxSerialized] (for security reasons). In that case Truncated will be set to true. +type ContractInvocation struct { + Hash util.Uint160 `json:"contract"` + Method string `json:"method"` + // Arguments are the arguments as passed to the `args` parameter of System.Contract.Call + // for use in the RPC Server and RPC Client. + Arguments *stackitem.Array `json:"arguments"` + // argumentsBytes is the serialized arguments used at the interop level. + argumentsBytes []byte + ArgumentsCount uint32 `json:"argumentscount"` + Truncated bool `json:"truncated"` +} + +// contractInvocationAux is an auxiliary struct for ContractInvocation JSON marshalling. +type contractInvocationAux struct { + Hash util.Uint160 `json:"hash"` + Method string `json:"method"` + Arguments json.RawMessage `json:"arguments,omitempty"` + ArgumentsCount uint32 `json:"argumentscount"` + Truncated bool `json:"truncated"` +} + +// NewContractInvocation returns a new ContractInvocation. +func NewContractInvocation(hash util.Uint160, method string, argBytes []byte, argCnt uint32) *ContractInvocation { + return &ContractInvocation{ + Hash: hash, + Method: method, + argumentsBytes: argBytes, + ArgumentsCount: argCnt, + Truncated: argBytes == nil, + } +} + +// DecodeBinary implements the Serializable interface. +func (ci *ContractInvocation) DecodeBinary(r *io.BinReader) { + ci.Hash.DecodeBinary(r) + ci.Method = r.ReadString() + ci.ArgumentsCount = r.ReadU32LE() + ci.Truncated = r.ReadBool() + if !ci.Truncated { + ci.argumentsBytes = r.ReadVarBytes() + } +} + +// EncodeBinary implements the Serializable interface. +func (ci *ContractInvocation) EncodeBinary(w *io.BinWriter) { + ci.EncodeBinaryWithContext(w, stackitem.NewSerializationContext()) +} + +// EncodeBinaryWithContext is the same as EncodeBinary, but allows to efficiently reuse +// stack item serialization context. +func (ci *ContractInvocation) EncodeBinaryWithContext(w *io.BinWriter, sc *stackitem.SerializationContext) { + ci.Hash.EncodeBinary(w) + w.WriteString(ci.Method) + w.WriteU32LE(ci.ArgumentsCount) + w.WriteBool(ci.Truncated) + if !ci.Truncated { + w.WriteVarBytes(ci.argumentsBytes) + } +} + +// MarshalJSON implements the json.Marshaler interface. +func (ci ContractInvocation) MarshalJSON() ([]byte, error) { + var item []byte + if ci.Arguments == nil && ci.argumentsBytes != nil { + si, err := stackitem.Deserialize(ci.argumentsBytes) + if err != nil { + return nil, err + } + item, err = stackitem.ToJSONWithTypes(si.(*stackitem.Array)) + if err != nil { + item = nil + } + } + return json.Marshal(contractInvocationAux{ + Hash: ci.Hash, + Method: ci.Method, + Arguments: item, + ArgumentsCount: ci.ArgumentsCount, + Truncated: ci.Truncated, + }) +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (ci *ContractInvocation) UnmarshalJSON(data []byte) error { + aux := new(contractInvocationAux) + if err := json.Unmarshal(data, aux); err != nil { + return err + } + var args *stackitem.Array + if aux.Arguments != nil { + arguments, err := stackitem.FromJSONWithTypes(aux.Arguments) + if err != nil { + return err + } + if t := arguments.Type(); t != stackitem.ArrayT { + return fmt.Errorf("failed to convert invocation state of type %s to array", t.String()) + } + args = arguments.(*stackitem.Array) + } + ci.Method = aux.Method + ci.Hash = aux.Hash + ci.ArgumentsCount = aux.ArgumentsCount + ci.Truncated = aux.Truncated + ci.Arguments = args + return nil +} diff --git a/pkg/core/state/contract_invocation_test.go b/pkg/core/state/contract_invocation_test.go new file mode 100644 index 000000000..eed8da370 --- /dev/null +++ b/pkg/core/state/contract_invocation_test.go @@ -0,0 +1,51 @@ +package state + +import ( + "testing" + + json "github.com/nspcc-dev/go-ordered-json" + "github.com/nspcc-dev/neo-go/internal/testserdes" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + "github.com/stretchr/testify/require" +) + +func TestContractInvocation_MarshalUnmarshalJSON(t *testing.T) { + t.Run("truncated", func(t *testing.T) { + ci := NewContractInvocation(util.Uint160{}, "fakeMethodCall", nil, 1) + testserdes.MarshalUnmarshalJSON(t, ci, new(ContractInvocation)) + }) + t.Run("not truncated", func(t *testing.T) { + si := stackitem.NewArray([]stackitem.Item{stackitem.NewBool(false)}) + argBytes, err := stackitem.NewSerializationContext().Serialize(si, false) + require.NoError(t, err) + + ci := NewContractInvocation(util.Uint160{}, "fakeMethodCall", argBytes, 1) + // Marshal and Unmarshal are asymmetric, test manually + out, err := json.Marshal(&ci) + require.NoError(t, err) + var ci2 ContractInvocation + err = json.Unmarshal(out, &ci2) + require.NoError(t, err) + require.Equal(t, ci.Hash, ci2.Hash) + require.Equal(t, ci.Method, ci2.Method) + require.Equal(t, ci.Truncated, ci2.Truncated) + require.Equal(t, ci.ArgumentsCount, ci2.ArgumentsCount) + require.Equal(t, si, ci2.Arguments) + }) +} + +func TestContractInvocation_EncodeDecodeBinary(t *testing.T) { + t.Run("truncated", func(t *testing.T) { + ci := NewContractInvocation(util.Uint160{}, "fakeMethodCall", nil, 1) + testserdes.EncodeDecodeBinary(t, ci, new(ContractInvocation)) + }) + t.Run("not truncated", func(t *testing.T) { + si := stackitem.NewArray([]stackitem.Item{stackitem.NewBool(false)}) + argBytes, err := stackitem.NewSerializationContext().Serialize(si, false) + require.NoError(t, err) + + ci := NewContractInvocation(util.Uint160{}, "fakeMethodCall", argBytes, 1) + testserdes.EncodeDecodeBinary(t, ci, new(ContractInvocation)) + }) +} diff --git a/pkg/core/state/notification_event.go b/pkg/core/state/notification_event.go index d30f231a9..3fde05f1e 100644 --- a/pkg/core/state/notification_event.go +++ b/pkg/core/state/notification_event.go @@ -12,6 +12,18 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" ) +const ( + // saveInvocationsBit acts as a marker using the VMState to indicate whether contract + // invocations (tracked by the VM) have been stored into the DB and thus whether they + // should be deserialized upon retrieval. This approach saves 1 byte for all + // applicationlogs over using WriteVarBytes. The original discussion can be found here + // https://github.com/nspcc-dev/neo-go/pull/3569#discussion_r1909357541 + saveInvocationsBit = 0x80 + // cleanSaveInvocationsBitMask is used to remove the save invocations marker bit from + // the VMState. + cleanSaveInvocationsBitMask = saveInvocationsBit ^ 0xFF +) + // NotificationEvent is a tuple of the scripthash that has emitted the Item as a // notification and the item itself. type NotificationEvent struct { @@ -78,6 +90,10 @@ func (aer *AppExecResult) EncodeBinary(w *io.BinWriter) { func (aer *AppExecResult) EncodeBinaryWithContext(w *io.BinWriter, sc *stackitem.SerializationContext) { w.WriteBytes(aer.Container[:]) w.WriteB(byte(aer.Trigger)) + invocLen := len(aer.Invocations) + if invocLen > 0 { + aer.VMState |= saveInvocationsBit + } w.WriteB(byte(aer.VMState)) w.WriteU64LE(uint64(aer.GasConsumed)) // Stack items are expected to be marshaled one by one. @@ -95,6 +111,12 @@ func (aer *AppExecResult) EncodeBinaryWithContext(w *io.BinWriter, sc *stackitem aer.Events[i].EncodeBinaryWithContext(w, sc) } w.WriteVarBytes([]byte(aer.FaultException)) + if invocLen > 0 { + w.WriteVarUint(uint64(invocLen)) + for i := range aer.Invocations { + aer.Invocations[i].EncodeBinaryWithContext(w, sc) + } + } } // DecodeBinary implements the Serializable interface. @@ -120,6 +142,10 @@ func (aer *AppExecResult) DecodeBinary(r *io.BinReader) { aer.Stack = arr r.ReadArray(&aer.Events) aer.FaultException = r.ReadString() + if aer.VMState&saveInvocationsBit != 0 { + r.ReadArray(&aer.Invocations) + aer.VMState &= cleanSaveInvocationsBitMask + } } // notificationEventAux is an auxiliary struct for NotificationEvent JSON marshalling. @@ -209,16 +235,18 @@ type Execution struct { Stack []stackitem.Item Events []NotificationEvent FaultException string + Invocations []ContractInvocation } // executionAux represents an auxiliary struct for Execution JSON marshalling. type executionAux struct { - Trigger string `json:"trigger"` - VMState string `json:"vmstate"` - GasConsumed int64 `json:"gasconsumed,string"` - Stack json.RawMessage `json:"stack"` - Events []NotificationEvent `json:"notifications"` - FaultException *string `json:"exception"` + Trigger string `json:"trigger"` + VMState string `json:"vmstate"` + GasConsumed int64 `json:"gasconsumed,string"` + Stack json.RawMessage `json:"stack"` + Events []NotificationEvent `json:"notifications"` + FaultException *string `json:"exception"` + Invocations []ContractInvocation `json:"invocations"` } // MarshalJSON implements the json.Marshaler interface. @@ -246,6 +274,7 @@ func (e Execution) MarshalJSON() ([]byte, error) { Stack: st, Events: e.Events, FaultException: exception, + Invocations: e.Invocations, }) } @@ -287,6 +316,7 @@ func (e *Execution) UnmarshalJSON(data []byte) error { if aux.FaultException != nil { e.FaultException = *aux.FaultException } + e.Invocations = aux.Invocations return nil }