From 124c3df2ff69638c8919d25512d99f8288e4e137 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 22 Aug 2023 15:57:55 +0300 Subject: [PATCH 1/6] rpcsrv: rename testcases related to unsupported state Testcase name should match the test purpose. Signed-off-by: Anna Shaleva --- pkg/services/rpcsrv/server_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/services/rpcsrv/server_test.go b/pkg/services/rpcsrv/server_test.go index b50a4be1f..5fa25ff44 100644 --- a/pkg/services/rpcsrv/server_test.go +++ b/pkg/services/rpcsrv/server_test.go @@ -98,7 +98,7 @@ var ( var rpcFunctionsWithUnsupportedStatesTestCases = map[string][]rpcTestCase{ "getproof": { { - name: "no params", + name: "unsupported state", params: `[]`, fail: true, errCode: neorpc.ErrUnsupportedStateCode, @@ -106,7 +106,7 @@ var rpcFunctionsWithUnsupportedStatesTestCases = map[string][]rpcTestCase{ }, "verifyproof": { { - name: "no params", + name: "unsupported state", params: `[]`, fail: true, errCode: neorpc.ErrUnsupportedStateCode, @@ -114,7 +114,7 @@ var rpcFunctionsWithUnsupportedStatesTestCases = map[string][]rpcTestCase{ }, "getstate": { { - name: "unknown root/item", + name: "unsupported state", params: `["0000000000000000000000000000000000000000000000000000000000000000", "` + testContractHash + `", "QQ=="]`, fail: true, errCode: neorpc.ErrUnsupportedStateCode, @@ -122,7 +122,7 @@ var rpcFunctionsWithUnsupportedStatesTestCases = map[string][]rpcTestCase{ }, "findstates": { { - name: "invalid contract", + name: "unsupported state", params: `["` + block20StateRootLE + `", "0xabcdef"]`, fail: true, errCode: neorpc.ErrUnsupportedStateCode, @@ -130,7 +130,7 @@ var rpcFunctionsWithUnsupportedStatesTestCases = map[string][]rpcTestCase{ }, "invokefunctionhistoric": { { - name: "no params", + name: "unsupported state", params: `[]`, fail: true, errCode: neorpc.ErrUnsupportedStateCode, From 617c628c242b3f1778c42c60f96a33344daf45d7 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 22 Aug 2023 19:00:43 +0300 Subject: [PATCH 2/6] rpcsrv, rpcclient: support `findstorage` and `findstoragehistoric` Close #3095 and add the corresponding historic extension. Signed-off-by: Anna Shaleva --- config/protocol.unit_testnet.yml | 1 + docs/node-configuration.md | 2 + pkg/config/config.go | 10 +- pkg/config/rpc_config.go | 23 +-- pkg/core/blockchain.go | 5 + pkg/neorpc/result/findstorage.go | 10 + pkg/rpcclient/rpc.go | 68 +++++++ pkg/rpcclient/rpc_test.go | 68 +++++++ pkg/services/rpcsrv/client_test.go | 128 +++++++++++++ pkg/services/rpcsrv/server.go | 173 ++++++++++++++--- pkg/services/rpcsrv/server_test.go | 296 +++++++++++++++++++++++++++++ 11 files changed, 741 insertions(+), 43 deletions(-) create mode 100644 pkg/neorpc/result/findstorage.go diff --git a/config/protocol.unit_testnet.yml b/config/protocol.unit_testnet.yml index 51def919e..0349b2064 100644 --- a/config/protocol.unit_testnet.yml +++ b/config/protocol.unit_testnet.yml @@ -66,6 +66,7 @@ ApplicationConfiguration: EnableCORSWorkaround: false SessionEnabled: true SessionExpirationTime: 2 # enough for tests as they run locally. + MaxFindStoragePageSize: 2 # small value to test server-side paging Prometheus: Enabled: false #since it's not useful for unit tests. Addresses: diff --git a/docs/node-configuration.md b/docs/node-configuration.md index 6ff7d8e47..77baa9a3a 100644 --- a/docs/node-configuration.md +++ b/docs/node-configuration.md @@ -202,6 +202,7 @@ RPC: MaxGasInvoke: 50 MaxIteratorResultItems: 100 MaxFindResultItems: 100 + MaxFindStoragePageSize: 50 MaxNEP11Tokens: 100 MaxWebSocketClients: 64 SessionEnabled: false @@ -238,6 +239,7 @@ where: `n`, only `n` iterations are returned and truncated is true, indicating that there is still data to be returned. - `MaxFindResultItems` - the maximum number of elements for `findstates` response. +- `MaxFindStoragePageSize` - the maximum number of elements for `findstorage` response per single page. - `MaxNEP11Tokens` - limit for the number of tokens returned from `getnep11balances` call. - `MaxWebSocketClients` - the maximum simultaneous websocket client connection diff --git a/pkg/config/config.go b/pkg/config/config.go index 93cc4ca43..ede5b4102 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -19,6 +19,9 @@ const ( // DefaultMaxIteratorResultItems is the default upper bound of traversed // iterator items per JSON-RPC response. DefaultMaxIteratorResultItems = 100 + // DefaultMaxFindStorageResultItems is the default maximum number of resulting + // contract storage items that can be retrieved by `findstorge` JSON-RPC handler. + DefaultMaxFindStorageResultItems = 50 ) // Version is the version of the node, set at the build time. @@ -71,9 +74,10 @@ func LoadFile(configPath string) (Config, error) { PingTimeout: 90 * time.Second, }, RPC: RPC{ - MaxIteratorResultItems: DefaultMaxIteratorResultItems, - MaxFindResultItems: 100, - MaxNEP11Tokens: 100, + MaxIteratorResultItems: DefaultMaxIteratorResultItems, + MaxFindResultItems: 100, + MaxFindStorageResultItems: DefaultMaxFindStorageResultItems, + MaxNEP11Tokens: 100, }, }, } diff --git a/pkg/config/rpc_config.go b/pkg/config/rpc_config.go index 44df2aa38..bd9d0acdd 100644 --- a/pkg/config/rpc_config.go +++ b/pkg/config/rpc_config.go @@ -11,17 +11,18 @@ type ( EnableCORSWorkaround bool `yaml:"EnableCORSWorkaround"` // MaxGasInvoke is the maximum amount of GAS which // can be spent during an RPC call. - MaxGasInvoke fixedn.Fixed8 `yaml:"MaxGasInvoke"` - MaxIteratorResultItems int `yaml:"MaxIteratorResultItems"` - MaxFindResultItems int `yaml:"MaxFindResultItems"` - MaxNEP11Tokens int `yaml:"MaxNEP11Tokens"` - MaxWebSocketClients int `yaml:"MaxWebSocketClients"` - SessionEnabled bool `yaml:"SessionEnabled"` - SessionExpirationTime int `yaml:"SessionExpirationTime"` - SessionBackedByMPT bool `yaml:"SessionBackedByMPT"` - SessionPoolSize int `yaml:"SessionPoolSize"` - StartWhenSynchronized bool `yaml:"StartWhenSynchronized"` - TLSConfig TLS `yaml:"TLSConfig"` + MaxGasInvoke fixedn.Fixed8 `yaml:"MaxGasInvoke"` + MaxIteratorResultItems int `yaml:"MaxIteratorResultItems"` + MaxFindResultItems int `yaml:"MaxFindResultItems"` + MaxFindStorageResultItems int `yaml:"MaxFindStoragePageSize"` + MaxNEP11Tokens int `yaml:"MaxNEP11Tokens"` + MaxWebSocketClients int `yaml:"MaxWebSocketClients"` + SessionEnabled bool `yaml:"SessionEnabled"` + SessionExpirationTime int `yaml:"SessionExpirationTime"` + SessionBackedByMPT bool `yaml:"SessionBackedByMPT"` + SessionPoolSize int `yaml:"SessionPoolSize"` + StartWhenSynchronized bool `yaml:"StartWhenSynchronized"` + TLSConfig TLS `yaml:"TLSConfig"` } // TLS describes SSL/TLS configuration. diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index e2327de60..6e71555a2 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -2133,6 +2133,11 @@ func (bc *Blockchain) GetStorageItem(id int32, key []byte) state.StorageItem { return bc.dao.GetStorageItem(id, key) } +// SeekStorage performs seek operation over contract storage. +func (bc *Blockchain) SeekStorage(id int32, prefix []byte, cont func(k, v []byte) bool) { + bc.dao.Seek(id, storage.SeekRange{Prefix: prefix}, cont) +} + // GetBlock returns a Block by the given hash. func (bc *Blockchain) GetBlock(hash util.Uint256) (*block.Block, error) { topBlock := bc.topBlock.Load() diff --git a/pkg/neorpc/result/findstorage.go b/pkg/neorpc/result/findstorage.go new file mode 100644 index 000000000..ad3bfe61f --- /dev/null +++ b/pkg/neorpc/result/findstorage.go @@ -0,0 +1,10 @@ +package result + +// FindStorage represents the result of `findstorage` RPC handler. +type FindStorage struct { + Results []KeyValue `json:"results"` + // Next contains the index of the next subsequent element of the contract storage + // that can be retrieved during the next iteration. + Next int `json:"next"` + Truncated bool `json:"truncated"` +} diff --git a/pkg/rpcclient/rpc.go b/pkg/rpcclient/rpc.go index 07cfbe077..8af451c60 100644 --- a/pkg/rpcclient/rpc.go +++ b/pkg/rpcclient/rpc.go @@ -559,6 +559,74 @@ func (c *Client) getStorage(params []any) ([]byte, error) { return resp, nil } +// FindStorageByHash returns contract storage items by the given contract hash and prefix. +// If `start` index is specified, items starting from `start` index are being returned +// (including item located at the start index). +func (c *Client) FindStorageByHash(contractHash util.Uint160, prefix []byte, start *int) (result.FindStorage, error) { + var params = []any{contractHash.StringLE(), prefix} + if start != nil { + params = append(params, *start) + } + return c.findStorage(params) +} + +// FindStorageByID returns contract storage items by the given contract ID and prefix. +// If `start` index is specified, items starting from `start` index are being returned +// (including item located at the start index). +func (c *Client) FindStorageByID(contractID int32, prefix []byte, start *int) (result.FindStorage, error) { + var params = []any{contractID, prefix} + if start != nil { + params = append(params, *start) + } + return c.findStorage(params) +} + +func (c *Client) findStorage(params []any) (result.FindStorage, error) { + var resp result.FindStorage + if err := c.performRequest("findstorage", params, &resp); err != nil { + return resp, err + } + return resp, nil +} + +// FindStorageByHashHistoric returns historical contract storage items by the given stateroot, +// historical contract hash and historical prefix. If `start` index is specified, then items +// starting from `start` index are being returned (including item located at the start index). +func (c *Client) FindStorageByHashHistoric(stateroot util.Uint256, historicalContractHash util.Uint160, historicalPrefix []byte, + start *int) (result.FindStorage, error) { + if historicalPrefix == nil { + historicalPrefix = []byte{} + } + var params = []any{stateroot.StringLE(), historicalContractHash.StringLE(), historicalPrefix} + if start != nil { + params = append(params, start) + } + return c.findStorageHistoric(params) +} + +// FindStorageByIDHistoric returns historical contract storage items by the given stateroot, +// historical contract ID and historical prefix. If `start` index is specified, then items +// starting from `start` index are being returned (including item located at the start index). +func (c *Client) FindStorageByIDHistoric(stateroot util.Uint256, historicalContractID int32, historicalPrefix []byte, + start *int) (result.FindStorage, error) { + if historicalPrefix == nil { + historicalPrefix = []byte{} + } + var params = []any{stateroot.StringLE(), historicalContractID, historicalPrefix} + if start != nil { + params = append(params, start) + } + return c.findStorageHistoric(params) +} + +func (c *Client) findStorageHistoric(params []any) (result.FindStorage, error) { + var resp result.FindStorage + if err := c.performRequest("findstoragehistoric", params, &resp); err != nil { + return resp, err + } + return resp, nil +} + // GetTransactionHeight returns the block index where the transaction is found. func (c *Client) GetTransactionHeight(hash util.Uint256) (uint32, error) { var ( diff --git a/pkg/rpcclient/rpc_test.go b/pkg/rpcclient/rpc_test.go index 590b82872..1a7cdbea2 100644 --- a/pkg/rpcclient/rpc_test.go +++ b/pkg/rpcclient/rpc_test.go @@ -915,6 +915,74 @@ var rpcClientTestCases = map[string][]rpcClientTestCase{ }, }, }, + "findstorage": { + { + name: "positive by hash", + invoke: func(c *Client) (any, error) { + cHash, _ := util.Uint160DecodeStringLE("5c9e40a12055c6b9e3f72271c9779958c842135d") + start := 1 + return c.FindStorageByHash(cHash, []byte("aa"), &start) + }, + serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"results":[{"key":"YWExMA==","value":"djI="}],"truncated":true, "next": 1}}`, + result: func(c *Client) any { + return result.FindStorage{ + Results: []result.KeyValue{{Key: []byte("aa10"), Value: []byte("v2")}}, + Truncated: true, + Next: 1, + } + }, + }, + { + name: "positive by ID", + invoke: func(c *Client) (any, error) { + start := 1 + return c.FindStorageByID(1, []byte("aa"), &start) + }, + serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"results":[{"key":"YWExMA==","value":"djI="}],"truncated":true, "next": 1}}`, + result: func(c *Client) any { + return result.FindStorage{ + Results: []result.KeyValue{{Key: []byte("aa10"), Value: []byte("v2")}}, + Truncated: true, + Next: 1, + } + }, + }, + }, + "findstoragehistoric": { + { + name: "positive by hash", + invoke: func(c *Client) (any, error) { + root, _ := util.Uint256DecodeStringLE("252e9d73d49c95c7618d40650da504e05183a1b2eed0685e42c360413c329170") + cHash, _ := util.Uint160DecodeStringLE("5c9e40a12055c6b9e3f72271c9779958c842135d") + start := 1 + return c.FindStorageByHashHistoric(root, cHash, []byte("aa"), &start) + }, + serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"results":[{"key":"YWExMA==","value":"djI="}],"truncated":true, "next": 1}}`, + result: func(c *Client) any { + return result.FindStorage{ + Results: []result.KeyValue{{Key: []byte("aa10"), Value: []byte("v2")}}, + Truncated: true, + Next: 1, + } + }, + }, + { + name: "positive by ID", + invoke: func(c *Client) (any, error) { + root, _ := util.Uint256DecodeStringLE("252e9d73d49c95c7618d40650da504e05183a1b2eed0685e42c360413c329170") + start := 1 + return c.FindStorageByIDHistoric(root, 1, []byte("aa"), &start) + }, + serverResponse: `{"id":1,"jsonrpc":"2.0","result":{"results":[{"key":"YWExMA==","value":"djI="}],"truncated":true, "next": 1}}`, + result: func(c *Client) any { + return result.FindStorage{ + Results: []result.KeyValue{{Key: []byte("aa10"), Value: []byte("v2")}}, + Truncated: true, + Next: 1, + } + }, + }, + }, "getstateheight": { { name: "positive", diff --git a/pkg/services/rpcsrv/client_test.go b/pkg/services/rpcsrv/client_test.go index a7e2b3784..2bbef0d5a 100644 --- a/pkg/services/rpcsrv/client_test.go +++ b/pkg/services/rpcsrv/client_test.go @@ -2562,3 +2562,131 @@ func TestActor_CallWithNilParam(t *testing.T) { require.True(t, strings.Contains(res.FaultException, "invalid conversion: Null/ByteString"), res.FaultException) } + +func TestClient_FindStorage(t *testing.T) { + chain, rpcSrv, httpSrv := initServerWithInMemoryChain(t) + defer chain.Close() + defer rpcSrv.Shutdown() + + c, err := rpcclient.New(context.Background(), httpSrv.URL, rpcclient.Options{}) + require.NoError(t, err) + require.NoError(t, c.Init()) + + h, err := util.Uint160DecodeStringLE(testContractHash) + require.NoError(t, err) + prefix := []byte("aa") + expected := result.FindStorage{ + Results: []result.KeyValue{ + { + Key: []byte("aa"), + Value: []byte("v1"), + }, + { + Key: []byte("aa10"), + Value: []byte("v2"), + }, + }, + Next: 2, + Truncated: true, + } + + // By hash. + actual, err := c.FindStorageByHash(h, prefix, nil) + require.NoError(t, err) + require.Equal(t, expected, actual) + + // By ID. + actual, err = c.FindStorageByID(1, prefix, nil) // Rubles contract + require.NoError(t, err) + require.Equal(t, expected, actual) + + // Non-nil start. + start := 1 + actual, err = c.FindStorageByHash(h, prefix, &start) + require.NoError(t, err) + require.Equal(t, result.FindStorage{ + Results: []result.KeyValue{ + { + Key: []byte("aa10"), + Value: []byte("v2"), + }, + { + Key: []byte("aa50"), + Value: []byte("v3"), + }, + }, + Next: 3, + Truncated: false, + }, actual) + + // Missing item. + actual, err = c.FindStorageByHash(h, []byte("unknown prefix"), nil) + require.NoError(t, err) + require.Equal(t, result.FindStorage{}, actual) +} + +func TestClient_FindStorageHistoric(t *testing.T) { + chain, rpcSrv, httpSrv := initServerWithInMemoryChain(t) + defer chain.Close() + defer rpcSrv.Shutdown() + + c, err := rpcclient.New(context.Background(), httpSrv.URL, rpcclient.Options{}) + require.NoError(t, err) + require.NoError(t, c.Init()) + + root, err := util.Uint256DecodeStringLE(block20StateRootLE) + require.NoError(t, err) + h, err := util.Uint160DecodeStringLE(testContractHash) + require.NoError(t, err) + prefix := []byte("aa") + expected := result.FindStorage{ + Results: []result.KeyValue{ + { + Key: []byte("aa10"), + Value: []byte("v2"), + }, + { + Key: []byte("aa50"), + Value: []byte("v3"), + }, + }, + Next: 2, + Truncated: true, + } + + // By hash. + actual, err := c.FindStorageByHashHistoric(root, h, prefix, nil) + require.NoError(t, err) + require.Equal(t, expected, actual) + + // By ID. + actual, err = c.FindStorageByIDHistoric(root, 1, prefix, nil) // Rubles contract + require.NoError(t, err) + require.Equal(t, expected, actual) + + // Non-nil start. + start := 1 + actual, err = c.FindStorageByHashHistoric(root, h, prefix, &start) + require.NoError(t, err) + require.Equal(t, result.FindStorage{ + Results: []result.KeyValue{ + { + Key: []byte("aa50"), + Value: []byte("v3"), + }, + { + Key: []byte("aa"), // order differs due to MPT traversal strategy. + Value: []byte("v1"), + }, + }, + Next: 3, + Truncated: false, + }, actual) + + // Missing item. + earlyRoot, err := chain.GetStateRoot(15) // there's no `aa10` value in Rubles contract by the moment of block #15 + require.NoError(t, err) + actual, err = c.FindStorageByHashHistoric(earlyRoot.Root, h, prefix, nil) + require.NoError(t, err) + require.Equal(t, result.FindStorage{}, actual) +} diff --git a/pkg/services/rpcsrv/server.go b/pkg/services/rpcsrv/server.go index 45d629278..865c963cd 100644 --- a/pkg/services/rpcsrv/server.go +++ b/pkg/services/rpcsrv/server.go @@ -51,6 +51,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest/standard" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/util/slice" "github.com/nspcc-dev/neo-go/pkg/vm" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" @@ -98,6 +99,7 @@ type ( GetValidators() ([]*keys.PublicKey, error) HeaderHeight() uint32 InitVerificationContext(ic *interop.Context, hash util.Uint160, witness *transaction.Witness) error + SeekStorage(id int32, prefix []byte, cont func(k, v []byte) bool) SubscribeForBlocks(ch chan *block.Block) SubscribeForExecutions(ch chan *state.AppExecResult) SubscribeForNotifications(ch chan *state.ContainedNotificationEvent) @@ -199,6 +201,8 @@ const ( var rpcHandlers = map[string]func(*Server, params.Params) (any, *neorpc.Error){ "calculatenetworkfee": (*Server).calculateNetworkFee, "findstates": (*Server).findStates, + "findstorage": (*Server).findStorage, + "findstoragehistoric": (*Server).findStorageHistoric, "getapplicationlog": (*Server).getApplicationLog, "getbestblockhash": (*Server).getBestBlockHash, "getblock": (*Server).getBlock, @@ -1414,17 +1418,25 @@ func (s *Server) getHash(contractID int32, cache map[int32]util.Uint160) (util.U return h, nil } -func (s *Server) contractIDFromParam(param *params.Param) (int32, *neorpc.Error) { +func (s *Server) contractIDFromParam(param *params.Param, root ...util.Uint256) (int32, *neorpc.Error) { var result int32 if param == nil { return 0, neorpc.ErrInvalidParams } if scriptHash, err := param.GetUint160FromHex(); err == nil { - cs := s.chain.GetContractState(scriptHash) - if cs == nil { - return 0, neorpc.ErrUnknownContract + if len(root) == 0 { + cs := s.chain.GetContractState(scriptHash) + if cs == nil { + return 0, neorpc.ErrUnknownContract + } + result = cs.ID + } else { + cs, respErr := s.getHistoricalContractState(root[0], scriptHash) + if respErr != nil { + return 0, respErr + } + result = cs.ID } - result = cs.ID } else { id, err := param.GetInt() if err != nil { @@ -1539,18 +1551,9 @@ func (s *Server) verifyProof(ps params.Params) (any, *neorpc.Error) { } func (s *Server) getState(ps params.Params) (any, *neorpc.Error) { - root, err := ps.Value(0).GetUint256() - if err != nil { - return nil, neorpc.WrapErrorWithData(neorpc.ErrInvalidParams, "invalid stateroot") - } - if s.chain.GetConfig().Ledger.KeepOnlyLatestState { - curr, err := s.chain.GetStateModule().GetStateRoot(s.chain.BlockHeight()) - if err != nil { - return nil, neorpc.NewInternalServerError(fmt.Sprintf("failed to get current stateroot: %s", err)) - } - if !curr.Root.Equals(root) { - return nil, neorpc.WrapErrorWithData(neorpc.ErrUnsupportedState, fmt.Sprintf("'getstate' is not supported for old states: %s", errKeepOnlyLatestState)) - } + root, respErr := s.getStateRootFromParam(ps.Value(0)) + if respErr != nil { + return nil, respErr } csHash, err := ps.Value(1).GetUint160FromHex() if err != nil { @@ -1576,18 +1579,9 @@ func (s *Server) getState(ps params.Params) (any, *neorpc.Error) { } func (s *Server) findStates(ps params.Params) (any, *neorpc.Error) { - root, err := ps.Value(0).GetUint256() - if err != nil { - return nil, neorpc.WrapErrorWithData(neorpc.ErrInvalidParams, "invalid stateroot") - } - if s.chain.GetConfig().Ledger.KeepOnlyLatestState { - curr, err := s.chain.GetStateModule().GetStateRoot(s.chain.BlockHeight()) - if err != nil { - return nil, neorpc.NewInternalServerError(fmt.Sprintf("failed to get current stateroot: %s", err)) - } - if !curr.Root.Equals(root) { - return nil, neorpc.WrapErrorWithData(neorpc.ErrUnsupportedState, fmt.Sprintf("'findstates' is not supported for old states: %s", errKeepOnlyLatestState)) - } + root, respErr := s.getStateRootFromParam(ps.Value(0)) + if respErr != nil { + return nil, respErr } csHash, err := ps.Value(1).GetUint160FromHex() if err != nil { @@ -1669,6 +1663,127 @@ func (s *Server) findStates(ps params.Params) (any, *neorpc.Error) { return res, nil } +// getStateRootFromParam retrieves state root hash from the provided parameter +// (only util.Uint256 serialized representation is allowed) and checks whether +// MPT states are supported for the old stateroot. +func (s *Server) getStateRootFromParam(p *params.Param) (util.Uint256, *neorpc.Error) { + root, err := p.GetUint256() + if err != nil { + return util.Uint256{}, neorpc.WrapErrorWithData(neorpc.ErrInvalidParams, "invalid stateroot") + } + if s.chain.GetConfig().Ledger.KeepOnlyLatestState { + curr, err := s.chain.GetStateModule().GetStateRoot(s.chain.BlockHeight()) + if err != nil { + return util.Uint256{}, neorpc.NewInternalServerError(fmt.Sprintf("failed to get current stateroot: %s", err)) + } + if !curr.Root.Equals(root) { + return util.Uint256{}, neorpc.WrapErrorWithData(neorpc.ErrUnsupportedState, fmt.Sprintf("state-based methods are not supported for old states: %s", errKeepOnlyLatestState)) + } + } + return root, nil +} + +func (s *Server) findStorage(reqParams params.Params) (any, *neorpc.Error) { + id, prefix, start, take, respErr := s.getFindStorageParams(reqParams) + if respErr != nil { + return nil, respErr + } + var ( + i int + end = start + take + res = new(result.FindStorage) + ) + s.chain.SeekStorage(id, prefix, func(k, v []byte) bool { + if i < start { + i++ + return true + } + if i < end { + res.Results = append(res.Results, result.KeyValue{ + Key: slice.Copy(append(prefix, k...)), // Don't strip prefix, as it is done in C#. + Value: v, + }) + i++ + return true + } + res.Truncated = true + return false + }) + res.Next = i + return res, nil +} + +func (s *Server) findStorageHistoric(reqParams params.Params) (any, *neorpc.Error) { + root, respErr := s.getStateRootFromParam(reqParams.Value(0)) + if respErr != nil { + return nil, respErr + } + if len(reqParams) < 2 { + return nil, neorpc.ErrInvalidParams + } + id, prefix, start, take, respErr := s.getFindStorageParams(reqParams[1:], root) + if respErr != nil { + return nil, respErr + } + + var ( + end = start + take + res = new(result.FindStorage) + pKey = makeStorageKey(id, prefix) + ) + // @roman-khimov, retrieving only the necessary part of the contract storage + // requires an mpt Billet refactoring, we can do it in a separate issue, create? + kvs, err := s.chain.GetStateModule().FindStates(root, pKey, nil, end+1) // +1 to define result truncation + if err != nil && !errors.Is(err, mpt.ErrNotFound) { + return nil, neorpc.NewInternalServerError(fmt.Sprintf("failed to find state items: %s", err)) + } + if len(kvs) == end+1 { + res.Truncated = true + kvs = kvs[:len(kvs)-1] + } + if start >= len(kvs) { + kvs = nil + } else { + kvs = kvs[start:] + } + + if len(kvs) != 0 { // keep consistency with `findstorage` response + res.Results = make([]result.KeyValue, len(kvs)) + for i := range res.Results { + res.Results[i] = result.KeyValue{ + Key: kvs[i].Key[4:], // Cut contract ID as it is done in C#. + Value: kvs[i].Value, + } + } + } + res.Next = start + len(res.Results) + return res, nil +} + +func (s *Server) getFindStorageParams(reqParams params.Params, root ...util.Uint256) (int32, []byte, int, int, *neorpc.Error) { + if len(reqParams) < 2 { + return 0, nil, 0, 0, neorpc.ErrInvalidParams + } + id, respErr := s.contractIDFromParam(reqParams.Value(0), root...) + if respErr != nil { + return 0, nil, 0, 0, respErr + } + + prefix, err := reqParams.Value(1).GetBytesBase64() + if err != nil { + return 0, nil, 0, 0, neorpc.WrapErrorWithData(neorpc.ErrInvalidParams, fmt.Sprintf("invalid prefix: %s", err)) + } + + var skip int + if len(reqParams) > 2 { + skip, err = reqParams.Value(2).GetInt() + if err != nil { + return 0, nil, 0, 0, neorpc.WrapErrorWithData(neorpc.ErrInvalidParams, fmt.Sprintf("invalid start: %s", err)) + } + } + return id, prefix, skip, s.config.MaxFindStorageResultItems, nil +} + func (s *Server) getHistoricalContractState(root util.Uint256, csHash util.Uint160) (*state.Contract, *neorpc.Error) { csKey := makeStorageKey(native.ManagementContractID, native.MakeContractKey(csHash)) csBytes, err := s.chain.GetStateModule().GetState(root, csKey) diff --git a/pkg/services/rpcsrv/server_test.go b/pkg/services/rpcsrv/server_test.go index 5fa25ff44..98ed281bf 100644 --- a/pkg/services/rpcsrv/server_test.go +++ b/pkg/services/rpcsrv/server_test.go @@ -128,6 +128,14 @@ var rpcFunctionsWithUnsupportedStatesTestCases = map[string][]rpcTestCase{ errCode: neorpc.ErrUnsupportedStateCode, }, }, + "findstoragehistoric": { + { + name: "unsupported state", + params: `["` + block20StateRootLE + `", "0xabcdef"]`, + fail: true, + errCode: neorpc.ErrUnsupportedStateCode, + }, + }, "invokefunctionhistoric": { { name: "unsupported state", @@ -651,6 +659,294 @@ var rpcTestCases = map[string][]rpcTestCase{ errCode: neorpc.InvalidParamsCode, }, }, + "findstorage": { + { + name: "not truncated", + params: fmt.Sprintf(`["%s", "%s"]`, testContractHash, base64.StdEncoding.EncodeToString([]byte("aa1"))), + result: func(_ *executor) any { return new(result.FindStorage) }, + check: func(t *testing.T, e *executor, res any) { + actual, ok := res.(*result.FindStorage) + require.True(t, ok) + + expected := &result.FindStorage{ + Results: []result.KeyValue{ + { + Key: []byte("aa10"), + Value: []byte("v2"), + }, + }, + Next: 1, + Truncated: false, + } + require.Equal(t, expected, actual) + }, + }, + { + name: "truncated first page", + params: fmt.Sprintf(`["%s", "%s"]`, testContractHash, base64.StdEncoding.EncodeToString([]byte("aa"))), + result: func(_ *executor) any { return new(result.FindStorage) }, + check: func(t *testing.T, e *executor, res any) { + actual, ok := res.(*result.FindStorage) + require.True(t, ok) + + expected := &result.FindStorage{ + Results: []result.KeyValue{ + { + Key: []byte("aa"), + Value: []byte("v1"), + }, + { + Key: []byte("aa10"), + Value: []byte("v2"), + }, + }, + Next: 2, + Truncated: true, + } + require.Equal(t, expected, actual) + }, + }, + { + name: "truncated second page", + params: fmt.Sprintf(`["%s", "%s", 2]`, testContractHash, base64.StdEncoding.EncodeToString([]byte("aa"))), + result: func(_ *executor) any { return new(result.FindStorage) }, + check: func(t *testing.T, e *executor, res any) { + actual, ok := res.(*result.FindStorage) + require.True(t, ok) + + expected := &result.FindStorage{ + Results: []result.KeyValue{ + { + Key: []byte("aa50"), + Value: []byte("v3"), + }, + }, + Next: 3, + Truncated: false, + } + require.Equal(t, expected, actual) + }, + }, + { + name: "empty prefix", + params: fmt.Sprintf(`["%s", ""]`, storageContractHash), + result: func(_ *executor) any { return new(result.FindStorage) }, + check: func(t *testing.T, e *executor, res any) { + actual, ok := res.(*result.FindStorage) + require.True(t, ok) + + expected := &result.FindStorage{ + Results: []result.KeyValue{ + { + Key: []byte{0x01, 0x00}, + Value: []byte{}, + }, + { + Key: []byte{0x01, 0x01}, + Value: []byte{0x01}, + }, + }, + Next: 2, + Truncated: true, + } + require.Equal(t, expected, actual) + }, + }, + { + name: "unknown key", + params: fmt.Sprintf(`["%s", "%s"]`, testContractHash, base64.StdEncoding.EncodeToString([]byte("unknown-key"))), + result: func(_ *executor) any { return new(result.FindStorage) }, + check: func(t *testing.T, e *executor, res any) { + actual, ok := res.(*result.FindStorage) + require.True(t, ok) + + expected := &result.FindStorage{ + Results: nil, + Next: 0, + Truncated: false, + } + require.Equal(t, expected, actual) + }, + }, + { + name: "no params", + params: `[]`, + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "no second parameter", + params: fmt.Sprintf(`["%s"]`, testContractHash), + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "invalid hash", + params: `["notahex"]`, + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "invalid key", + params: fmt.Sprintf(`["%s", "notabase64$"]`, testContractHash), + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "invalid page", + params: fmt.Sprintf(`["%s", "", "not-an-int"]`, testContractHash), + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + }, + "findstoragehistoric": { + { + name: "not truncated", + params: fmt.Sprintf(`["%s", "%s", "%s"]`, block20StateRootLE, testContractHash, base64.StdEncoding.EncodeToString([]byte("aa1"))), + result: func(_ *executor) any { return new(result.FindStorage) }, + check: func(t *testing.T, e *executor, res any) { + actual, ok := res.(*result.FindStorage) + require.True(t, ok) + + expected := &result.FindStorage{ + Results: []result.KeyValue{ + { + Key: []byte("aa10"), + Value: []byte("v2"), + }, + }, + Next: 1, + Truncated: false, + } + require.Equal(t, expected, actual) + }, + }, + { + name: "truncated first page", + params: fmt.Sprintf(`["%s", "%s", "%s"]`, block20StateRootLE, testContractHash, base64.StdEncoding.EncodeToString([]byte("aa"))), + result: func(_ *executor) any { return new(result.FindStorage) }, + check: func(t *testing.T, e *executor, res any) { + actual, ok := res.(*result.FindStorage) + require.True(t, ok) + + expected := &result.FindStorage{ + Results: []result.KeyValue{ + { + Key: []byte("aa10"), // items traversal order may differ from the one provided by `findstorage` due to MPT traversal strategy. + Value: []byte("v2"), + }, + { + Key: []byte("aa50"), + Value: []byte("v3"), + }, + }, + Next: 2, + Truncated: true, + } + require.Equal(t, expected, actual) + }, + }, + { + name: "truncated second page", + params: fmt.Sprintf(`["%s","%s", "%s", 2]`, block20StateRootLE, testContractHash, base64.StdEncoding.EncodeToString([]byte("aa"))), + result: func(_ *executor) any { return new(result.FindStorage) }, + check: func(t *testing.T, e *executor, res any) { + actual, ok := res.(*result.FindStorage) + require.True(t, ok) + + expected := &result.FindStorage{ + Results: []result.KeyValue{ + { + Key: []byte("aa"), + Value: []byte("v1"), + }, + }, + Next: 3, + Truncated: false, + } + require.Equal(t, expected, actual) + }, + }, + { + name: "empty prefix", + params: fmt.Sprintf(`["%s", "%s", ""]`, block20StateRootLE, nnsContractHash), + result: func(_ *executor) any { return new(result.FindStorage) }, + check: func(t *testing.T, e *executor, res any) { + actual, ok := res.(*result.FindStorage) + require.True(t, ok) + + expected := &result.FindStorage{ + Results: []result.KeyValue{ + { + Key: []byte{0x00}, // total supply + Value: []byte{0x01}, + }, + { + Key: append([]byte{0x01}, testchain.PrivateKeyByID(0).GetScriptHash().BytesBE()...), // balance of priv0 + Value: []byte{0x01}, + }, + }, + Next: 2, + Truncated: true, + } + require.Equal(t, expected, actual) + }, + }, + { + name: "unknown key", + params: fmt.Sprintf(`["%s", "%s", "%s"]`, block20StateRootLE, testContractHash, base64.StdEncoding.EncodeToString([]byte("unknown-key"))), + result: func(_ *executor) any { return new(result.FindStorage) }, + check: func(t *testing.T, e *executor, res any) { + actual, ok := res.(*result.FindStorage) + require.True(t, ok) + + expected := &result.FindStorage{} + require.Equal(t, expected, actual) + }, + }, + { + name: "no params", + params: `[]`, + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "invalid stateroot", + params: `[12345]`, + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "no second parameter", + params: fmt.Sprintf(`["%s"]`, block20StateRootLE), + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "no third parameter", + params: fmt.Sprintf(`["%s", "%s"]`, block20StateRootLE, testContractHash), + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "invalid hash", + params: fmt.Sprintf(`["%s", "notahex"]`, block20StateRootLE), + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "invalid key", + params: fmt.Sprintf(`["%s", "%s", "notabase64$"]`, block20StateRootLE, testContractHash), + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "invalid page", + params: fmt.Sprintf(`["%s", "%s", "", "not-an-int"]`, block20StateRootLE, testContractHash), + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + }, "getbestblockhash": { { params: "[]", From 97d523ceed9dcba915120ed17a4d8da19e237c4b Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 23 Aug 2023 17:09:45 +0300 Subject: [PATCH 3/6] docs: fix typo Signed-off-by: Anna Shaleva --- docs/node-configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/node-configuration.md b/docs/node-configuration.md index 77baa9a3a..e7088c979 100644 --- a/docs/node-configuration.md +++ b/docs/node-configuration.md @@ -281,7 +281,7 @@ where: pool, then invocation result will contain corresponding error inside the `FaultException` field. - `StartWhenSynchronized` controls when RPC server will be started, by default - (`false` setting) it's started immediately and RPC is availabe during node + (`false` setting) it's started immediately and RPC is available during node synchronization. Setting it to `true` will make the node start RPC service only after full synchronization. - `TLS` section configures TLS protocol. From 1fb0c96e2cdabc958ec064f9faf6862ce9e39142 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 22 Aug 2023 19:29:22 +0300 Subject: [PATCH 4/6] rpcsrv, rpcclient: support `getstoragehistoric` call Make it similar to `findstoragehistoric`. Signed-off-by: Anna Shaleva --- pkg/rpcclient/rpc.go | 20 ++++++++++++ pkg/rpcclient/rpc_test.go | 44 +++++++++++++++++++++++++ pkg/services/rpcsrv/client_test.go | 33 +++++++++++++++++++ pkg/services/rpcsrv/server.go | 31 ++++++++++++++++++ pkg/services/rpcsrv/server_test.go | 52 ++++++++++++++++++++++++++++++ 5 files changed, 180 insertions(+) diff --git a/pkg/rpcclient/rpc.go b/pkg/rpcclient/rpc.go index 8af451c60..0a7215558 100644 --- a/pkg/rpcclient/rpc.go +++ b/pkg/rpcclient/rpc.go @@ -559,6 +559,26 @@ func (c *Client) getStorage(params []any) ([]byte, error) { return resp, nil } +// GetStorageByIDHistoric returns the historical stored value according to the +// contract ID and, stored key and specified stateroot. +func (c *Client) GetStorageByIDHistoric(root util.Uint256, id int32, key []byte) ([]byte, error) { + return c.getStorageHistoric([]any{root.StringLE(), id, key}) +} + +// GetStorageByHashHistoric returns the historical stored value according to the +// contract script hash, the stored key and specified stateroot. +func (c *Client) GetStorageByHashHistoric(root util.Uint256, hash util.Uint160, key []byte) ([]byte, error) { + return c.getStorageHistoric([]any{root.StringLE(), hash.StringLE(), key}) +} + +func (c *Client) getStorageHistoric(params []any) ([]byte, error) { + var resp []byte + if err := c.performRequest("getstoragehistoric", params, &resp); err != nil { + return nil, err + } + return resp, nil +} + // FindStorageByHash returns contract storage items by the given contract hash and prefix. // If `start` index is specified, items starting from `start` index are being returned // (including item located at the start index). diff --git a/pkg/rpcclient/rpc_test.go b/pkg/rpcclient/rpc_test.go index 1a7cdbea2..5562a21ac 100644 --- a/pkg/rpcclient/rpc_test.go +++ b/pkg/rpcclient/rpc_test.go @@ -1040,6 +1040,50 @@ var rpcClientTestCases = map[string][]rpcClientTestCase{ }, }, }, + "getstoragehistoric": { + { + name: "by hash, positive", + invoke: func(c *Client) (any, error) { + root, _ := util.Uint256DecodeStringLE("252e9d73d49c95c7618d40650da504e05183a1b2eed0685e42c360413c329170") + hash, err := util.Uint160DecodeStringLE("03febccf81ac85e3d795bc5cbd4e84e907812aa3") + if err != nil { + panic(err) + } + key, err := hex.DecodeString("5065746572") + if err != nil { + panic(err) + } + return c.GetStorageByHashHistoric(root, hash, key) + }, + serverResponse: `{"jsonrpc":"2.0","id":1,"result":"TGlu"}`, + result: func(c *Client) any { + value, err := hex.DecodeString("4c696e") + if err != nil { + panic(err) + } + return value + }, + }, + { + name: "by ID, positive", + invoke: func(c *Client) (any, error) { + root, _ := util.Uint256DecodeStringLE("252e9d73d49c95c7618d40650da504e05183a1b2eed0685e42c360413c329170") + key, err := hex.DecodeString("5065746572") + if err != nil { + panic(err) + } + return c.GetStorageByIDHistoric(root, -1, key) + }, + serverResponse: `{"jsonrpc":"2.0","id":1,"result":"TGlu"}`, + result: func(c *Client) any { + value, err := hex.DecodeString("4c696e") + if err != nil { + panic(err) + } + return value + }, + }, + }, "gettransactionheight": { { name: "positive", diff --git a/pkg/services/rpcsrv/client_test.go b/pkg/services/rpcsrv/client_test.go index 2bbef0d5a..0e3b0dd6d 100644 --- a/pkg/services/rpcsrv/client_test.go +++ b/pkg/services/rpcsrv/client_test.go @@ -2690,3 +2690,36 @@ func TestClient_FindStorageHistoric(t *testing.T) { require.NoError(t, err) require.Equal(t, result.FindStorage{}, actual) } + +func TestClient_GetStorageHistoric(t *testing.T) { + chain, rpcSrv, httpSrv := initServerWithInMemoryChain(t) + defer chain.Close() + defer rpcSrv.Shutdown() + + c, err := rpcclient.New(context.Background(), httpSrv.URL, rpcclient.Options{}) + require.NoError(t, err) + require.NoError(t, c.Init()) + + root, err := util.Uint256DecodeStringLE(block20StateRootLE) + require.NoError(t, err) + h, err := util.Uint160DecodeStringLE(testContractHash) + require.NoError(t, err) + key := []byte("aa10") + expected := []byte("v2") + + // By hash. + actual, err := c.GetStorageByHashHistoric(root, h, key) + require.NoError(t, err) + require.Equal(t, expected, actual) + + // By ID. + actual, err = c.GetStorageByIDHistoric(root, 1, key) // Rubles contract + require.NoError(t, err) + require.Equal(t, expected, actual) + + // Missing item. + earlyRoot, err := chain.GetStateRoot(15) // there's no `aa10` value in Rubles contract by the moment of block #15 + require.NoError(t, err) + _, err = c.GetStorageByHashHistoric(earlyRoot.Root, h, key) + require.ErrorIs(t, neorpc.ErrUnknownStorageItem, err) +} diff --git a/pkg/services/rpcsrv/server.go b/pkg/services/rpcsrv/server.go index 865c963cd..5c936e7d9 100644 --- a/pkg/services/rpcsrv/server.go +++ b/pkg/services/rpcsrv/server.go @@ -229,6 +229,7 @@ var rpcHandlers = map[string]func(*Server, params.Params) (any, *neorpc.Error){ "getstateheight": (*Server).getStateHeight, "getstateroot": (*Server).getStateRoot, "getstorage": (*Server).getStorage, + "getstoragehistoric": (*Server).getStorageHistoric, "gettransactionheight": (*Server).getTransactionHeight, "getunclaimedgas": (*Server).getUnclaimedGas, "getnextblockvalidators": (*Server).getNextBlockValidators, @@ -1855,6 +1856,36 @@ func (s *Server) getStorage(ps params.Params) (any, *neorpc.Error) { return []byte(item), nil } +func (s *Server) getStorageHistoric(ps params.Params) (any, *neorpc.Error) { + root, respErr := s.getStateRootFromParam(ps.Value(0)) + if respErr != nil { + return nil, respErr + } + if len(ps) < 2 { + return nil, neorpc.ErrInvalidParams + } + + id, rErr := s.contractIDFromParam(ps.Value(1), root) + if rErr != nil { + return nil, rErr + } + key, err := ps.Value(2).GetBytesBase64() + if err != nil { + return nil, neorpc.ErrInvalidParams + } + pKey := makeStorageKey(id, key) + + v, err := s.chain.GetStateModule().GetState(root, pKey) + if err != nil && !errors.Is(err, mpt.ErrNotFound) { + return nil, neorpc.NewInternalServerError(fmt.Sprintf("failed to get state item: %s", err)) + } + if v == nil { + return "", neorpc.ErrUnknownStorageItem + } + + return v, nil +} + func (s *Server) getrawtransaction(reqParams params.Params) (any, *neorpc.Error) { txHash, err := reqParams.Value(0).GetUint256() if err != nil { diff --git a/pkg/services/rpcsrv/server_test.go b/pkg/services/rpcsrv/server_test.go index 98ed281bf..b85640d25 100644 --- a/pkg/services/rpcsrv/server_test.go +++ b/pkg/services/rpcsrv/server_test.go @@ -659,6 +659,58 @@ var rpcTestCases = map[string][]rpcTestCase{ errCode: neorpc.InvalidParamsCode, }, }, + "getstoragehistoric": { + { + name: "positive", + params: fmt.Sprintf(`["%s", "%s", "%s"]`, block20StateRootLE, testContractHash, base64.StdEncoding.EncodeToString([]byte("aa10"))), + result: func(e *executor) any { + v := base64.StdEncoding.EncodeToString([]byte("v2")) + return &v + }, + }, + { + name: "missing key", + params: fmt.Sprintf(`["%s", "%s", "dGU="]`, block20StateRootLE, testContractHash), + fail: true, + errCode: neorpc.ErrUnknownStorageItemCode, + }, + { + name: "no params", + params: `[]`, + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "no second parameter", + params: fmt.Sprintf(`["%s"]`, block20StateRootLE), + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "no third parameter", + params: fmt.Sprintf(`["%s", "%s"]`, block20StateRootLE, testContractHash), + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "invalid stateroot", + params: `["notahex"]`, + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "invalid hash", + params: fmt.Sprintf(`["%s", "notahex"]`, block20StateRootLE), + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + { + name: "invalid key", + params: fmt.Sprintf(`["%s", "%s", "notabase64$"]`, block20StateRootLE, testContractHash), + fail: true, + errCode: neorpc.InvalidParamsCode, + }, + }, "findstorage": { { name: "not truncated", From c7d5ee08985597622ce2ee00ac77581afedfcb98 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 22 Aug 2023 19:46:07 +0300 Subject: [PATCH 5/6] docs: refactor historical RPC calls documentation 1. Reorder paragraphs. 2. Extend common historical calls description. 2. Document `getstoragehistoric` and `findstoragehistoric` extensions. Signed-off-by: Anna Shaleva --- docs/rpc.md | 45 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/docs/rpc.md b/docs/rpc.md index 13083e8e1..2fd0777a6 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -37,6 +37,7 @@ which would yield the response: | ------- | | `calculatenetworkfee` | | `findstates` | +| `findstorage` | | `getapplicationlog` | | `getbestblockhash` | | `getblock` | @@ -237,7 +238,30 @@ block. It can be removed in future versions, but at the moment you can use it to see how much GAS is burned with a particular block (because system fees are burned). -#### `invokecontractverifyhistoric`, `invokefunctionhistoric` and `invokescripthistoric` calls +#### Historic calls + +A set of `*historic` extension methods provide the ability of interacting with +*historical* chain state including invoking contract methods, running scripts and +retrieving contract storage items. It means that the contracts' storage state has +all its values got from MPT with the specified stateroot from past (or, which is +the same, with the stateroot of the block of the specified height). All +operations related to the contract storage will be performed using this past +contracts' storage state and using interop context (if required by the RPC +handler) with a block which is next to the block with the specified height. + +Any historical RPC call needs the historical chain state to be presented in the +node storage, thus if the node keeps only latest MPT state the historical call +can not be handled properly and +[neorpc.ErrUnsupportedState](https://github.com/nspcc-dev/neo-go/blob/87e4b6beaafa3c180184cbbe88ba143378c5024c/pkg/neorpc/errors.go#L134) +is returned in this case. The historical calls only guaranteed to correctly work +on archival node that stores all MPT data. If a node keeps the number of latest +states and has the GC on (this setting corresponds to the +`RemoveUntraceableBlocks` set to `true`), then the behaviour of historical RPC +call is undefined. GC can always kick some data out of the storage while the +historical call is executing, thus keep in mind that the call can be processed +with `RemoveUntraceableBlocks` only with limitations on available data. + +##### `invokecontractverifyhistoric`, `invokefunctionhistoric` and `invokescripthistoric` calls These methods provide the ability of *historical* calls and accept block hash or block index or stateroot hash as the first parameter and the list of parameters @@ -250,16 +274,15 @@ the block with the specified height. This allows to perform test invocation usin the specified past chain state. These methods may be useful for debugging purposes. -Behavior note: any historical RPC call need the historical chain state to be -presented in the node storage, thus if the node keeps only latest MPT state -the historical call can not be handled properly.The historical calls only -guaranteed to correctly work on archival node that stores all MPT data. If a -node keeps the number of latest states and has the GC on (this setting -corresponds to the `RemoveUntraceableBlocks` set to `true`), then the behaviour -of historical RPC call is undefined. GC can always kick some data out of the -storage while the historical call is executing, thus keep in mind that the call -can be processed with `RemoveUntraceableBlocks` only with limitations on -available data. +##### `getstoragehistoric` and `findstoragehistoric` calls + +These methods provide the ability of retrieving *historical* contract storage +items and accept stateroot hash as the first parameter and the list of parameters +that is the same as of `getstorage` and `findstorage` correspondingly. The +historical storage items retrieval process assume that the contracts' storage +state has all its values got from MPT with the specified stateroot. This allows +to track the contract storage scheme using the specified past chain state. These +methods may be useful for debugging purposes. #### `submitnotaryrequest` call From 7b64b693bd3e59b7990692441029ee0ea3ac3507 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Thu, 24 Aug 2023 14:23:42 +0300 Subject: [PATCH 6/6] rpcsrv: refactor `findstoragehistoric` handler to avoid DoS attack Do not retrieve the whole set of storage items when trying to find the ones from the specified start. Use DAO's Seek interface implemented over MPT TrieStore to retrieve only the necessary items. Signed-off-by: Anna Shaleva --- pkg/core/blockchain.go | 1 + pkg/core/stateroot/module.go | 26 +++++++++++++++ pkg/services/rpcsrv/server.go | 59 +++++++++++++++-------------------- 3 files changed, 53 insertions(+), 33 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 6e71555a2..30b68c70c 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -195,6 +195,7 @@ type StateRoot interface { CurrentLocalStateRoot() util.Uint256 CurrentValidatedHeight() uint32 FindStates(root util.Uint256, prefix, start []byte, max int) ([]storage.KeyValue, error) + SeekStates(root util.Uint256, prefix []byte, f func(k, v []byte) bool) GetState(root util.Uint256, key []byte) ([]byte, error) GetStateProof(root util.Uint256, key []byte) ([][]byte, error) GetStateRoot(height uint32) (*state.MPTRoot, error) diff --git a/pkg/core/stateroot/module.go b/pkg/core/stateroot/module.go index 442164d8b..61b1d54db 100644 --- a/pkg/core/stateroot/module.go +++ b/pkg/core/stateroot/module.go @@ -93,6 +93,32 @@ func (s *Module) FindStates(root util.Uint256, prefix, start []byte, max int) ([ return tr.Find(prefix, start, max) } +// SeekStates traverses over contract storage with the state based on the +// specified root. `prefix` is expected to consist of contract ID and the desired +// storage items prefix. `cont` is called for every matching key-value pair; +// the resulting key does not include contract ID and the desired storage item +// prefix (they are stripped to match the Blockchain's SeekStorage behaviour. +// The result includes item with the key that equals to the `prefix` (if +// such item is found in the storage). Traversal process is stopped when `false` +// is returned from `cont`. +func (s *Module) SeekStates(root util.Uint256, prefix []byte, cont func(k, v []byte) bool) { + // Allow accessing old values, it's RO thing. + store := mpt.NewTrieStore(root, s.mode&^mpt.ModeGCFlag, storage.NewMemCachedStore(s.Store)) + + // Tiny hack to satisfy TrieStore with the given prefix. This + // storage.STStorage prefix is a stub that will be stripped by the + // TrieStore.Seek while performing MPT traversal and isn't actually relevant + // here. + key := make([]byte, len(prefix)+1) + key[0] = byte(storage.STStorage) + copy(key[1:], prefix) + + store.Seek(storage.SeekRange{Prefix: key}, func(k, v []byte) bool { + // Cut the prefix to match the Blockchain's SeekStorage behaviour. + return cont(k[len(key):], v) + }) +} + // GetStateProof returns proof of having key in the MPT with the specified root. func (s *Module) GetStateProof(root util.Uint256, key []byte) ([][]byte, error) { // Allow accessing old values, it's RO thing. diff --git a/pkg/services/rpcsrv/server.go b/pkg/services/rpcsrv/server.go index 5c936e7d9..a8bd13c45 100644 --- a/pkg/services/rpcsrv/server.go +++ b/pkg/services/rpcsrv/server.go @@ -99,7 +99,6 @@ type ( GetValidators() ([]*keys.PublicKey, error) HeaderHeight() uint32 InitVerificationContext(ic *interop.Context, hash util.Uint160, witness *transaction.Witness) error - SeekStorage(id int32, prefix []byte, cont func(k, v []byte) bool) SubscribeForBlocks(ch chan *block.Block) SubscribeForExecutions(ch chan *state.AppExecResult) SubscribeForNotifications(ch chan *state.ContainedNotificationEvent) @@ -111,6 +110,13 @@ type ( VerifyTx(*transaction.Transaction) error VerifyWitness(util.Uint160, hash.Hashable, *transaction.Witness, int64) (int64, error) mempool.Feer // fee interface + ContractStorageSeeker + } + + // ContractStorageSeeker is the interface `findstorage*` handlers need to be able to + // seek over contract storage. + ContractStorageSeeker interface { + SeekStorage(id int32, prefix []byte, cont func(k, v []byte) bool) } // OracleHandler is the interface oracle service needs to provide for the Server. @@ -1689,12 +1695,16 @@ func (s *Server) findStorage(reqParams params.Params) (any, *neorpc.Error) { if respErr != nil { return nil, respErr } + return s.findStorageInternal(id, prefix, start, take, s.chain) +} + +func (s *Server) findStorageInternal(id int32, prefix []byte, start, take int, seeker ContractStorageSeeker) (any, *neorpc.Error) { var ( i int end = start + take res = new(result.FindStorage) ) - s.chain.SeekStorage(id, prefix, func(k, v []byte) bool { + seeker.SeekStorage(id, prefix, func(k, v []byte) bool { if i < start { i++ return true @@ -1727,38 +1737,21 @@ func (s *Server) findStorageHistoric(reqParams params.Params) (any, *neorpc.Erro return nil, respErr } - var ( - end = start + take - res = new(result.FindStorage) - pKey = makeStorageKey(id, prefix) - ) - // @roman-khimov, retrieving only the necessary part of the contract storage - // requires an mpt Billet refactoring, we can do it in a separate issue, create? - kvs, err := s.chain.GetStateModule().FindStates(root, pKey, nil, end+1) // +1 to define result truncation - if err != nil && !errors.Is(err, mpt.ErrNotFound) { - return nil, neorpc.NewInternalServerError(fmt.Sprintf("failed to find state items: %s", err)) - } - if len(kvs) == end+1 { - res.Truncated = true - kvs = kvs[:len(kvs)-1] - } - if start >= len(kvs) { - kvs = nil - } else { - kvs = kvs[start:] - } + return s.findStorageInternal(id, prefix, start, take, mptStorageSeeker{ + root: root, + module: s.chain.GetStateModule(), + }) +} - if len(kvs) != 0 { // keep consistency with `findstorage` response - res.Results = make([]result.KeyValue, len(kvs)) - for i := range res.Results { - res.Results[i] = result.KeyValue{ - Key: kvs[i].Key[4:], // Cut contract ID as it is done in C#. - Value: kvs[i].Value, - } - } - } - res.Next = start + len(res.Results) - return res, nil +// mptStorageSeeker is an auxiliary structure that implements ContractStorageSeeker interface. +type mptStorageSeeker struct { + root util.Uint256 + module core.StateRoot +} + +func (s mptStorageSeeker) SeekStorage(id int32, prefix []byte, cont func(k, v []byte) bool) { + key := makeStorageKey(id, prefix) + s.module.SeekStates(s.root, key, cont) } func (s *Server) getFindStorageParams(reqParams params.Params, root ...util.Uint256) (int32, []byte, int, int, *neorpc.Error) {