Merge pull request #2783 from nspcc-dev/rpcbindings-iterators

Initial iterator support for RPC bindings

cli/smartcontract/contract_test.go

@@ -1004,6 +1004,9 @@ func TestCompileExamples(t *testing.T) {
 
 			outF := filepath.Join(tmpDir, info.Name()+".nef")
 			manifestF := filepath.Join(tmpDir, info.Name()+".manifest.json")
+			bindingF := filepath.Join(tmpDir, info.Name()+".binding.yml")
+			wrapperF := filepath.Join(tmpDir, info.Name()+".go")
+			rpcWrapperF := filepath.Join(tmpDir, info.Name()+".rpc.go")
 
 			cfgName := filterFilename(infos, ".yml")
 			opts := []string{
@@ -1012,6 +1015,7 @@ func TestCompileExamples(t *testing.T) {
 				"--out", outF,
 				"--manifest", manifestF,
 				"--config", filepath.Join(examplePath, info.Name(), cfgName),
+				"--bindings", bindingF,
 			}
 			e.Run(t, opts...)
 
@@ -1030,6 +1034,16 @@ func TestCompileExamples(t *testing.T) {
 				require.NotNil(t, m.ABI.GetMethod("put", 1))
 				require.NotNil(t, m.ABI.GetMethod("put", 2))
 			}
+			e.Run(t, "neo-go", "contract", "generate-wrapper",
+				"--manifest", manifestF,
+				"--config", bindingF,
+				"--out", wrapperF,
+				"--hash", "0x00112233445566778899aabbccddeeff00112233")
+			e.Run(t, "neo-go", "contract", "generate-rpcwrapper",
+				"--manifest", manifestF,
+				"--config", bindingF,
+				"--out", rpcWrapperF,
+				"--hash", "0x00112233445566778899aabbccddeeff00112233")
 		})
 	}
 

cli/smartcontract/generate_test.go

@@ -360,6 +360,9 @@ func TestGenerateRPCBindings(t *testing.T) {
 	checkBinding(filepath.Join("testdata", "verifyrpc", "verify.manifest.json"),
 		"0x00112233445566778899aabbccddeeff00112233",
 		filepath.Join("testdata", "verifyrpc", "verify.go"))
+	checkBinding(filepath.Join("testdata", "nonepiter", "iter.manifest.json"),
+		"0x00112233445566778899aabbccddeeff00112233",
+		filepath.Join("testdata", "nonepiter", "iter.go"))
 }
 
 func TestGenerate_Errors(t *testing.T) {

cli/smartcontract/testdata/nameservice/nns.go

@@ -2,7 +2,9 @@
 package nameservice
 
 import (
+	"github.com/google/uuid"
 	"github.com/nspcc-dev/neo-go/pkg/core/transaction"
+	"github.com/nspcc-dev/neo-go/pkg/neorpc/result"
 	"github.com/nspcc-dev/neo-go/pkg/rpcclient/nep11"
 	"github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap"
 	"github.com/nspcc-dev/neo-go/pkg/smartcontract"
@@ -59,8 +61,17 @@ func New(actor Actor) *Contract {
 
 
 // Roots invokes `roots` method of contract.
-func (c *ContractReader) Roots() (stackitem.Item, error) {
+func (c *ContractReader) Roots() (uuid.UUID, result.Iterator, error) {
-	return unwrap.Item(c.invoker.Call(Hash, "roots"))
+	return unwrap.SessionIterator(c.invoker.Call(Hash, "roots"))
+}
+
+// RootsExpanded is similar to Roots (uses the same contract
+// method), but can be useful if the server used doesn't support sessions and
+// doesn't expand iterators. It creates a script that will get the specified
+// number of result items from the iterator right in the VM and return them to
+// you. It's only limited by VM stack and GAS available for RPC invocations.
+func (c *ContractReader) RootsExpanded(_numOfIteratorItems int) ([]stackitem.Item, error) {
+	return unwrap.Array(c.invoker.CallAndExpandIterator(Hash, "roots", _numOfIteratorItems))
 }
 
 // GetPrice invokes `getPrice` method of contract.
@@ -79,8 +90,17 @@ func (c *ContractReader) GetRecord(name string, typev *big.Int) (string, error)
 }
 
 // GetAllRecords invokes `getAllRecords` method of contract.
-func (c *ContractReader) GetAllRecords(name string) (stackitem.Item, error) {
+func (c *ContractReader) GetAllRecords(name string) (uuid.UUID, result.Iterator, error) {
-	return unwrap.Item(c.invoker.Call(Hash, "getAllRecords", name))
+	return unwrap.SessionIterator(c.invoker.Call(Hash, "getAllRecords", name))
+}
+
+// GetAllRecordsExpanded is similar to GetAllRecords (uses the same contract
+// method), but can be useful if the server used doesn't support sessions and
+// doesn't expand iterators. It creates a script that will get the specified
+// number of result items from the iterator right in the VM and return them to
+// you. It's only limited by VM stack and GAS available for RPC invocations.
+func (c *ContractReader) GetAllRecordsExpanded(name string, _numOfIteratorItems int) ([]stackitem.Item, error) {
+	return unwrap.Array(c.invoker.CallAndExpandIterator(Hash, "getAllRecords", _numOfIteratorItems, name))
 }
 
 // Resolve invokes `resolve` method of contract.

cli/smartcontract/testdata/nonepiter/iter.go

@@ -0,0 +1,60 @@
+// Package nonnepxxcontractwithiterators contains RPC wrappers for Non-NEPXX contract with iterators contract.
+package nonnepxxcontractwithiterators
+
+import (
+	"github.com/google/uuid"
+	"github.com/nspcc-dev/neo-go/pkg/neorpc/result"
+	"github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap"
+	"github.com/nspcc-dev/neo-go/pkg/util"
+	"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
+)
+
+// Hash contains contract hash.
+var Hash = util.Uint160{0x33, 0x22, 0x11, 0x0, 0xff, 0xee, 0xdd, 0xcc, 0xbb, 0xaa, 0x99, 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x0}
+
+// Invoker is used by ContractReader to call various safe methods.
+type Invoker interface {
+	Call(contract util.Uint160, operation string, params ...interface{}) (*result.Invoke, error)
+	CallAndExpandIterator(contract util.Uint160, method string, maxItems int, params ...interface{}) (*result.Invoke, error)
+	TerminateSession(sessionID uuid.UUID) error
+	TraverseIterator(sessionID uuid.UUID, iterator *result.Iterator, num int) ([]stackitem.Item, error)
+}
+
+// ContractReader implements safe contract methods.
+type ContractReader struct {
+	invoker Invoker
+}
+
+// NewReader creates an instance of ContractReader using Hash and the given Invoker.
+func NewReader(invoker Invoker) *ContractReader {
+	return &ContractReader{invoker}
+}
+
+
+// Tokens invokes `tokens` method of contract.
+func (c *ContractReader) Tokens() (uuid.UUID, result.Iterator, error) {
+	return unwrap.SessionIterator(c.invoker.Call(Hash, "tokens"))
+}
+
+// TokensExpanded is similar to Tokens (uses the same contract
+// method), but can be useful if the server used doesn't support sessions and
+// doesn't expand iterators. It creates a script that will get the specified
+// number of result items from the iterator right in the VM and return them to
+// you. It's only limited by VM stack and GAS available for RPC invocations.
+func (c *ContractReader) TokensExpanded(_numOfIteratorItems int) ([]stackitem.Item, error) {
+	return unwrap.Array(c.invoker.CallAndExpandIterator(Hash, "tokens", _numOfIteratorItems))
+}
+
+// GetAllRecords invokes `getAllRecords` method of contract.
+func (c *ContractReader) GetAllRecords(name string) (uuid.UUID, result.Iterator, error) {
+	return unwrap.SessionIterator(c.invoker.Call(Hash, "getAllRecords", name))
+}
+
+// GetAllRecordsExpanded is similar to GetAllRecords (uses the same contract
+// method), but can be useful if the server used doesn't support sessions and
+// doesn't expand iterators. It creates a script that will get the specified
+// number of result items from the iterator right in the VM and return them to
+// you. It's only limited by VM stack and GAS available for RPC invocations.
+func (c *ContractReader) GetAllRecordsExpanded(name string, _numOfIteratorItems int) ([]stackitem.Item, error) {
+	return unwrap.Array(c.invoker.CallAndExpandIterator(Hash, "getAllRecords", _numOfIteratorItems, name))
+}

cli/smartcontract/testdata/nonepiter/iter.manifest.json

@@ -0,0 +1,33 @@
+{
+   "groups" : [],
+   "abi" : {
+      "events" : [],
+      "methods" : [
+         {
+            "parameters" : [],
+            "safe" : true,
+            "name" : "tokens",
+            "offset" : 0,
+            "returntype" : "InteropInterface"
+         },
+         {
+            "offset" : 1,
+            "returntype" : "InteropInterface",
+            "safe" : true,
+            "parameters" : [
+               {
+                  "type" : "String",
+                  "name" : "name"
+               }
+            ],
+            "name" : "getAllRecords"
+         }
+      ]
+   },
+   "supportedstandards" : [],
+   "trusts" : [],
+   "extra" : {},
+   "permissions" : [],
+   "name" : "Non-NEPXX contract with iterators",
+   "features" : {}
+}

docs/compiler.md

@@ -449,7 +449,10 @@ returns false.
 If your contract is NEP-11 or NEP-17 that's autodetected and an appropriate
 package is included as well. Notice that the type data available in the
 manifest is limited, so in some cases the interface generated may use generic
-stackitem types. Iterators are not supported yet.
+stackitem types. Any InteropInterface returned from a method is treated as
+iterator and an appropriate unwrapper is used with UUID and iterator structure
+result. This pair can then be used in Invoker `TraverseIterator` method to
+retrieve actual resulting items.
 
 ```
 $ ./bin/neo-go contract generate-rpcwrapper --manifest manifest.json --out rpcwrapper.go --hash 0x1b4357bff5a01bdf2a6581247cf9ed1e24629176

pkg/smartcontract/binding/generate.go

@@ -79,6 +79,8 @@ type (
 	}
 )
 
+var srcTemplate = template.Must(template.New("generate").Parse(srcTmpl))
+
 // NewConfig initializes and returns a new config instance.
 func NewConfig() Config {
 	return Config{
@@ -88,21 +90,15 @@ func NewConfig() Config {
 }
 
 // Generate writes Go file containing smartcontract bindings to the `cfg.Output`.
+// It doesn't check manifest from Config for validity, incorrect manifest can
+// lead to unexpected results.
 func Generate(cfg Config) error {
-	ctr, err := TemplateFromManifest(cfg, scTypeToGo)
+	ctr := TemplateFromManifest(cfg, scTypeToGo)
-	if err != nil {
-		return err
-	}
 	ctr.Imports = append(ctr.Imports, "github.com/nspcc-dev/neo-go/pkg/interop/contract")
 	ctr.Imports = append(ctr.Imports, "github.com/nspcc-dev/neo-go/pkg/interop/neogointernal")
 	sort.Strings(ctr.Imports)
 
-	tmp, err := template.New("generate").Parse(srcTmpl)
+	return srcTemplate.Execute(cfg.Output, ctr)
-	if err != nil {
-		return err
-	}
-
-	return tmp.Execute(cfg.Output, ctr)
 }
 
 func scTypeToGo(name string, typ smartcontract.ParamType, overrides map[string]Override) (string, string) {
@@ -143,8 +139,9 @@ func scTypeToGo(name string, typ smartcontract.ParamType, overrides map[string]O
 }
 
 // TemplateFromManifest create a contract template using the given configuration
-// and type conversion function.
+// and type conversion function. It assumes manifest to be present in the
-func TemplateFromManifest(cfg Config, scTypeConverter func(string, smartcontract.ParamType, map[string]Override) (string, string)) (ContractTmpl, error) {
+// configuration and assumes it to be correct (passing IsValid check).
+func TemplateFromManifest(cfg Config, scTypeConverter func(string, smartcontract.ParamType, map[string]Override) (string, string)) ContractTmpl {
 	hStr := ""
 	for _, b := range cfg.Hash.BytesBE() {
 		hStr += fmt.Sprintf("\\x%02x", b)
@@ -206,10 +203,6 @@ func TemplateFromManifest(cfg Config, scTypeConverter func(string, smartcontract
 		var varnames = make(map[string]bool)
 		for i := range m.Parameters {
 			name := m.Parameters[i].Name
-			if name == "" {
-				return ctr, fmt.Errorf("manifest ABI method %q/%d: parameter #%d is unnamed", m.Name, len(m.Parameters), i)
-			}
-
 			typeStr, pkg := scTypeConverter(m.Name+"."+name, m.Parameters[i].Type, cfg.Overrides)
 			if pkg != "" {
 				imports[pkg] = struct{}{}
@@ -239,7 +232,7 @@ func TemplateFromManifest(cfg Config, scTypeConverter func(string, smartcontract
 		ctr.Imports = append(ctr.Imports, imp)
 	}
 
-	return ctr, nil
+	return ctr
 }
 
 func upperFirst(s string) string {

pkg/smartcontract/rpcbinding/binding.go

@@ -25,6 +25,17 @@ func (c *ContractReader) {{.Name}}({{range $index, $arg := .Arguments -}}
 		{{- range $arg := .Arguments -}}, {{.Name}}{{end}})
 	{{- end}}
 }
+{{- if eq .CallFlag "SessionIterator"}}
+
+// {{.Name}}Expanded is similar to {{.Name}} (uses the same contract
+// method), but can be useful if the server used doesn't support sessions and
+// doesn't expand iterators. It creates a script that will get the specified
+// number of result items from the iterator right in the VM and return them to
+// you. It's only limited by VM stack and GAS available for RPC invocations.
+func (c *ContractReader) {{.Name}}Expanded({{range $index, $arg := .Arguments}}{{.Name}} {{.Type}}, {{end}}_numOfIteratorItems int) ([]stackitem.Item, error) {
+	return unwrap.Array(c.invoker.CallAndExpandIterator(Hash, "{{.NameABI}}", _numOfIteratorItems{{range $arg := .Arguments}}, {{.Name}}{{end}}))
+}
+{{- end -}}
 {{- end -}}
 {{- define "METHOD" -}}
 {{- if eq .ReturnType "bool"}}func scriptFor{{.Name}}({{range $index, $arg := .Arguments -}}
@@ -93,9 +104,15 @@ var Hash = {{ .Hash }}
 {{if .HasReader}}// Invoker is used by ContractReader to call various safe methods.
 type Invoker interface {
 {{if or .IsNep11D .IsNep11ND}}	nep11.Invoker
-{{else if .IsNep17}}	nep17.Invoker
+{{else -}}
+{{ if .IsNep17}}	nep17.Invoker
 {{else if len .SafeMethods}}	Call(contract util.Uint160, operation string, params ...interface{}) (*result.Invoke, error)
 {{end -}}
+{{if .HasIterator}}	CallAndExpandIterator(contract util.Uint160, method string, maxItems int, params ...interface{}) (*result.Invoke, error)
+	TerminateSession(sessionID uuid.UUID) error
+	TraverseIterator(sessionID uuid.UUID, iterator *result.Iterator, num int) ([]stackitem.Item, error)
+{{end -}}
+{{end -}}
 }
 
 {{end -}}
@@ -196,8 +213,9 @@ type (
 		IsNep11ND bool
 		IsNep17   bool
 
-		HasReader bool
+		HasReader   bool
-		HasWriter bool
+		HasWriter   bool
+		HasIterator bool
 	}
 )
 
@@ -207,6 +225,8 @@ func NewConfig() binding.Config {
 }
 
 // Generate writes Go file containing smartcontract bindings to the `cfg.Output`.
+// It doesn't check manifest from Config for validity, incorrect manifest can
+// lead to unexpected results.
 func Generate(cfg binding.Config) error {
 	// Avoid changing *cfg.Manifest.
 	mfst := *cfg.Manifest
@@ -246,11 +266,7 @@ func Generate(cfg binding.Config) error {
 		mfst.ABI.Methods = dropStdMethods(mfst.ABI.Methods, standard.Nep17Payable)
 	}
 
-	bctr, err := binding.TemplateFromManifest(cfg, scTypeToGo)
+	ctr.ContractTmpl = binding.TemplateFromManifest(cfg, scTypeToGo)
-	if err != nil {
-		return err
-	}
-	ctr.ContractTmpl = bctr
 	ctr = scTemplateToRPC(cfg, ctr, imports)
 
 	return srcTemplate.Execute(cfg.Output, ctr)
@@ -334,9 +350,19 @@ func scTemplateToRPC(cfg binding.Config, ctr ContractTmpl, imports map[string]st
 	for i := range ctr.SafeMethods {
 		switch ctr.SafeMethods[i].ReturnType {
 		case "interface{}":
-			imports["github.com/nspcc-dev/neo-go/pkg/vm/stackitem"] = struct{}{}
+			abim := cfg.Manifest.ABI.GetMethod(ctr.SafeMethods[i].NameABI, len(ctr.SafeMethods[i].Arguments))
-			ctr.SafeMethods[i].ReturnType = "stackitem.Item"
+			if abim.ReturnType == smartcontract.InteropInterfaceType {
-			ctr.SafeMethods[i].CallFlag = "Item"
+				imports["github.com/google/uuid"] = struct{}{}
+				imports["github.com/nspcc-dev/neo-go/pkg/vm/stackitem"] = struct{}{}
+				imports["github.com/nspcc-dev/neo-go/pkg/neorpc/result"] = struct{}{}
+				ctr.SafeMethods[i].ReturnType = "uuid.UUID, result.Iterator"
+				ctr.SafeMethods[i].CallFlag = "SessionIterator"
+				ctr.HasIterator = true
+			} else {
+				imports["github.com/nspcc-dev/neo-go/pkg/vm/stackitem"] = struct{}{}
+				ctr.SafeMethods[i].ReturnType = "stackitem.Item"
+				ctr.SafeMethods[i].CallFlag = "Item"
+			}
 		case "bool":
 			ctr.SafeMethods[i].CallFlag = "Bool"
 		case "*big.Int":