Merge pull request #811 from nspcc-dev/feature/debug

compiler: support neo-debugger

cli/smartcontract/smart_contract.go

@@ -92,6 +92,10 @@ func NewCommands() []cli.Command {
 						Name:  "debug, d",
 						Usage: "Debug mode will print out additional information after a compiling",
 					},
+					cli.StringFlag{
+						Name:  "emitdebug",
+						Usage: "Emit debug info in a separate file",
+					},
 				},
 			},
 			{
@@ -348,6 +352,8 @@ func contractCompile(ctx *cli.Context) error {
 	o := &compiler.Options{
 		Outfile: ctx.String("out"),
 		Debug:   ctx.Bool("debug"),
+
+		DebugInfo: ctx.String("emitdebug"),
 	}
 
 	result, err := compiler.CompileAndSave(src, o)

pkg/compiler/codegen.go

@@ -18,6 +18,7 @@ import (
 	"github.com/nspcc-dev/neo-go/pkg/vm"
 	"github.com/nspcc-dev/neo-go/pkg/vm/emit"
 	"github.com/nspcc-dev/neo-go/pkg/vm/opcode"
+	"golang.org/x/tools/go/loader"
 )
 
 // The identifier of the entry function. Default set to Main.
@@ -51,6 +52,11 @@ type codegen struct {
 	// A label to be used in the next statement.
 	nextLabel string
 
+	// sequencePoints is mapping from method name to a slice
+	// containing info about mapping from opcode's offset
+	// to a text span in the source file.
+	sequencePoints map[string][]DebugSeqPoint
+
 	// Label table for recording jump destinations.
 	l []int
 }
@@ -211,6 +217,7 @@ func (c *codegen) convertFuncDecl(file ast.Node, decl *ast.FuncDecl) {
 		f = c.newFunc(decl)
 	}
 
+	f.rng.Start = uint16(c.prog.Len())
 	c.scope = f
 	ast.Inspect(decl, c.scope.analyzeVoidCalls) // @OPTIMIZE
 
@@ -256,10 +263,13 @@ func (c *codegen) convertFuncDecl(file ast.Node, decl *ast.FuncDecl) {
 
 	// If this function returns the void (no return stmt) we will cleanup its junk on the stack.
 	if !hasReturnStmt(decl) {
+		c.saveSequencePoint(decl.Body)
 		emit.Opcode(c.prog.BinWriter, opcode.FROMALTSTACK)
 		emit.Opcode(c.prog.BinWriter, opcode.DROP)
 		emit.Opcode(c.prog.BinWriter, opcode.RET)
 	}
+
+	f.rng.End = uint16(c.prog.Len() - 1)
 }
 
 func (c *codegen) Visit(node ast.Node) ast.Visitor {
@@ -276,21 +286,25 @@ func (c *codegen) Visit(node ast.Node) ast.Visitor {
 		for _, spec := range n.Specs {
 			switch t := spec.(type) {
 			case *ast.ValueSpec:
+				for _, id := range t.Names {
+					c.scope.newLocal(id.Name)
+					c.registerDebugVariable(id.Name, t.Type)
+				}
 				if len(t.Values) != 0 {
 					for i, val := range t.Values {
 						ast.Walk(c, val)
-						l := c.scope.newLocal(t.Names[i].Name)
+						l := c.scope.loadLocal(t.Names[i].Name)
 						c.emitStoreLocal(l)
 					}
 				} else if c.isCompoundArrayType(t.Type) {
 					emit.Opcode(c.prog.BinWriter, opcode.PUSH0)
 					emit.Opcode(c.prog.BinWriter, opcode.NEWARRAY)
-					l := c.scope.newLocal(t.Names[0].Name)
+					l := c.scope.loadLocal(t.Names[0].Name)
 					c.emitStoreLocal(l)
 				} else if n, ok := c.isStructType(t.Type); ok {
 					emit.Int(c.prog.BinWriter, int64(n))
 					emit.Opcode(c.prog.BinWriter, opcode.NEWSTRUCT)
-					l := c.scope.newLocal(t.Names[0].Name)
+					l := c.scope.loadLocal(t.Names[0].Name)
 					c.emitStoreLocal(l)
 				}
 			}
@@ -299,7 +313,7 @@ func (c *codegen) Visit(node ast.Node) ast.Visitor {
 
 	case *ast.AssignStmt:
 		multiRet := len(n.Rhs) != len(n.Lhs)
-
+		c.saveSequencePoint(n)
 		for i := 0; i < len(n.Lhs); i++ {
 			switch t := n.Lhs[i].(type) {
 			case *ast.Ident:
@@ -310,6 +324,11 @@ func (c *codegen) Visit(node ast.Node) ast.Visitor {
 					c.convertToken(n.Tok)
 					l := c.scope.loadLocal(t.Name)
 					c.emitStoreLocal(l)
+				case token.DEFINE:
+					if !multiRet {
+						c.registerDebugVariable(t.Name, n.Rhs[i])
+					}
+					fallthrough
 				default:
 					if i == 0 || !multiRet {
 						ast.Walk(c, n.Rhs[i])
@@ -403,6 +422,7 @@ func (c *codegen) Visit(node ast.Node) ast.Visitor {
 			ast.Walk(c, n.Results[i])
 		}
 
+		c.saveSequencePoint(n)
 		emit.Opcode(c.prog.BinWriter, opcode.FROMALTSTACK)
 		emit.Opcode(c.prog.BinWriter, opcode.DROP) // Cleanup the stack.
 		emit.Opcode(c.prog.BinWriter, opcode.RET)
@@ -644,6 +664,8 @@ func (c *codegen) Visit(node ast.Node) ast.Visitor {
 			return nil
 		}
 
+		c.saveSequencePoint(n)
+
 		args := transformArgs(n.Fun, n.Args)
 
 		// Handle the arguments
@@ -1237,23 +1259,12 @@ func (c *codegen) newFunc(decl *ast.FuncDecl) *funcScope {
 	return f
 }
 
-// CodeGen compiles the program to bytecode.
+func (c *codegen) compile(info *buildInfo, pkg *loader.PackageInfo) error {
-func CodeGen(info *buildInfo) ([]byte, error) {
-	pkg := info.program.Package(info.initialPackage)
-	c := &codegen{
-		buildInfo: info,
-		prog:      io.NewBufBinWriter(),
-		l:         []int{},
-		funcs:     map[string]*funcScope{},
-		labels:    map[labelWithType]uint16{},
-		typeInfo:  &pkg.Info,
-	}
-
 	// Resolve the entrypoint of the program.
 	main, mainFile := resolveEntryPoint(mainIdent, pkg)
 	if main == nil {
 		c.prog.Err = fmt.Errorf("could not find func main. Did you forget to declare it? ")
-		return []byte{}, c.prog.Err
+		return c.prog.Err
 	}
 
 	funUsage := analyzeFuncUsage(info.program.AllPackages)
@@ -1294,14 +1305,36 @@ func CodeGen(info *buildInfo) ([]byte, error) {
 		}
 	}
 
-	if c.prog.Err != nil {
+	return c.prog.Err
-		return nil, c.prog.Err
 }
+
+func newCodegen(info *buildInfo, pkg *loader.PackageInfo) *codegen {
+	return &codegen{
+		buildInfo: info,
+		prog:      io.NewBufBinWriter(),
+		l:         []int{},
+		funcs:     map[string]*funcScope{},
+		labels:    map[labelWithType]uint16{},
+		typeInfo:  &pkg.Info,
+
+		sequencePoints: make(map[string][]DebugSeqPoint),
+	}
+}
+
+// CodeGen compiles the program to bytecode.
+func CodeGen(info *buildInfo) ([]byte, *DebugInfo, error) {
+	pkg := info.program.Package(info.initialPackage)
+	c := newCodegen(info, pkg)
+
+	if err := c.compile(info, pkg); err != nil {
+		return nil, nil, err
+	}
+
 	buf := c.prog.Bytes()
 	if err := c.writeJumps(buf); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return buf, nil
+	return buf, c.emitDebugInfo(), nil
 }
 
 func (c *codegen) resolveFuncDecls(f *ast.File) {

pkg/compiler/compiler.go

@@ -2,11 +2,13 @@ package compiler
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"go/parser"
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"golang.org/x/tools/go/loader"
@@ -22,6 +24,9 @@ type Options struct {
 	// The name of the output file.
 	Outfile string
 
+	// The name of the output for debug info.
+	DebugInfo string
+
 	// Debug outputs a hex encoded string of the generated bytecode.
 	Debug bool
 }
@@ -31,10 +36,9 @@ type buildInfo struct {
 	program        *loader.Program
 }
 
-// Compile compiles a Go program into bytecode that can run on the NEO virtual machine.
+func getBuildInfo(src interface{}) (*buildInfo, error) {
-func Compile(r io.Reader) ([]byte, error) {
 	conf := loader.Config{ParserMode: parser.ParseComments}
-	f, err := conf.ParseFile("", r)
+	f, err := conf.ParseFile("", src)
 	if err != nil {
 		return nil, err
 	}
@@ -45,12 +49,15 @@ func Compile(r io.Reader) ([]byte, error) {
 		return nil, err
 	}
 
-	ctx := &buildInfo{
+	return &buildInfo{
 		initialPackage: f.Name.Name,
 		program:        prog,
+	}, nil
 }
 
-	buf, err := CodeGen(ctx)
+// Compile compiles a Go program into bytecode that can run on the NEO virtual machine.
+func Compile(r io.Reader) ([]byte, error) {
+	buf, _, err := CompileWithDebugInfo(r)
 	if err != nil {
 		return nil, err
 	}
@@ -58,6 +65,15 @@ func Compile(r io.Reader) ([]byte, error) {
 	return buf, nil
 }
 
+// CompileWithDebugInfo compiles a Go program into bytecode and emits debug info.
+func CompileWithDebugInfo(r io.Reader) ([]byte, *DebugInfo, error) {
+	ctx, err := getBuildInfo(r)
+	if err != nil {
+		return nil, nil, err
+	}
+	return CodeGen(ctx)
+}
+
 // CompileAndSave will compile and save the file to disk.
 func CompileAndSave(src string, o *Options) ([]byte, error) {
 	if !strings.HasSuffix(src, ".go") {
@@ -74,11 +90,23 @@ func CompileAndSave(src string, o *Options) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	b, err = Compile(bytes.NewReader(b))
+	b, di, err := CompileWithDebugInfo(bytes.NewReader(b))
 	if err != nil {
 		return nil, fmt.Errorf("error while trying to compile smart contract file: %v", err)
 	}
-
 	out := fmt.Sprintf("%s.%s", o.Outfile, o.Ext)
-	return b, ioutil.WriteFile(out, b, os.ModePerm)
+	err = ioutil.WriteFile(out, b, os.ModePerm)
+	if o.DebugInfo == "" {
+		return b, err
+	}
+	p, err := filepath.Abs(src)
+	if err != nil {
+		return b, err
+	}
+	di.Documents = append(di.Documents, p)
+	data, err := json.Marshal(di)
+	if err != nil {
+		return b, err
+	}
+	return b, ioutil.WriteFile(o.DebugInfo, data, os.ModePerm)
 }

pkg/compiler/debug.go

@@ -0,0 +1,262 @@
+package compiler
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"go/ast"
+	"go/types"
+	"strconv"
+	"strings"
+)
+
+// DebugInfo represents smart-contract debug information.
+type DebugInfo struct {
+	EntryPoint string            `json:"entrypoint"`
+	Documents  []string          `json:"documents"`
+	Methods    []MethodDebugInfo `json:"methods"`
+	Events     []EventDebugInfo  `json:"events"`
+}
+
+// MethodDebugInfo represents smart-contract's method debug information.
+type MethodDebugInfo struct {
+	ID string `json:"id"`
+	// Name is the name of the method together with the namespace it belongs to.
+	Name DebugMethodName `json:"name"`
+	// Range is the range of smart-contract's opcodes corresponding to the method.
+	Range DebugRange `json:"range"`
+	// Parameters is a list of method's parameters.
+	Parameters []DebugParam `json:"params"`
+	// ReturnType is method's return type.
+	ReturnType string   `json:"return-type"`
+	Variables  []string `json:"variables"`
+	// SeqPoints is a map between source lines and byte-code instruction offsets.
+	SeqPoints []DebugSeqPoint `json:"sequence-points"`
+}
+
+// DebugMethodName is a combination of a namespace and name.
+type DebugMethodName struct {
+	Namespace string
+	Name      string
+}
+
+// EventDebugInfo represents smart-contract's event debug information.
+type EventDebugInfo struct {
+	ID string `json:"id"`
+	// Name is a human-readable event name in a format "{namespace}-{name}".
+	Name       string       `json:"name"`
+	Parameters []DebugParam `json:"parameters"`
+}
+
+// DebugSeqPoint represents break-point for debugger.
+type DebugSeqPoint struct {
+	// Opcode is an opcode's address.
+	Opcode int
+	// Document is an index of file where sequence point occurs.
+	Document int
+	// StartLine is the first line of the break-pointed statement.
+	StartLine int
+	// StartCol is the first column of the break-pointed statement.
+	StartCol int
+	// EndLine is the last line of the break-pointed statement.
+	EndLine int
+	// EndCol is the last column of the break-pointed statement.
+	EndCol int
+}
+
+// DebugRange represents method's section in bytecode.
+type DebugRange struct {
+	Start uint16
+	End   uint16
+}
+
+// DebugParam represents variables's name and type.
+type DebugParam struct {
+	Name string
+	Type string
+}
+
+func (c *codegen) saveSequencePoint(n ast.Node) {
+	fset := c.buildInfo.program.Fset
+	start := fset.Position(n.Pos())
+	end := fset.Position(n.End())
+	c.sequencePoints[c.scope.name] = append(c.sequencePoints[c.scope.name], DebugSeqPoint{
+		Opcode:    c.prog.Len(),
+		StartLine: start.Line,
+		StartCol:  start.Offset,
+		EndLine:   end.Line,
+		EndCol:    end.Offset,
+	})
+}
+
+func (c *codegen) emitDebugInfo() *DebugInfo {
+	d := &DebugInfo{
+		EntryPoint: mainIdent,
+		Events:     []EventDebugInfo{},
+	}
+	for name, scope := range c.funcs {
+		m := c.methodInfoFromScope(name, scope)
+		if m.Range.Start == m.Range.End {
+			continue
+		}
+		d.Methods = append(d.Methods, *m)
+	}
+	return d
+}
+
+func (c *codegen) registerDebugVariable(name string, expr ast.Expr) {
+	typ := c.scTypeFromExpr(expr)
+	c.scope.variables = append(c.scope.variables, name+","+typ)
+}
+
+func (c *codegen) methodInfoFromScope(name string, scope *funcScope) *MethodDebugInfo {
+	ps := scope.decl.Type.Params
+	params := make([]DebugParam, 0, ps.NumFields())
+	for i := range params {
+		for j := range ps.List[i].Names {
+			params = append(params, DebugParam{
+				Name: ps.List[i].Names[j].Name,
+				Type: c.scTypeFromExpr(ps.List[i].Type),
+			})
+		}
+	}
+	return &MethodDebugInfo{
+		ID:         name,
+		Name:       DebugMethodName{Name: name},
+		Range:      scope.rng,
+		Parameters: params,
+		ReturnType: c.scReturnTypeFromScope(scope),
+		SeqPoints:  c.sequencePoints[name],
+		Variables:  scope.variables,
+	}
+}
+
+func (c *codegen) scReturnTypeFromScope(scope *funcScope) string {
+	results := scope.decl.Type.Results
+	switch results.NumFields() {
+	case 0:
+		return "Void"
+	case 1:
+		return c.scTypeFromExpr(results.List[0].Type)
+	default:
+		// multiple return values are not supported in debugger
+		return "Any"
+	}
+}
+
+func (c *codegen) scTypeFromExpr(typ ast.Expr) string {
+	switch t := c.typeInfo.Types[typ].Type.(type) {
+	case *types.Basic:
+		info := t.Info()
+		switch {
+		case info&types.IsInteger != 0:
+			return "Integer"
+		case info&types.IsBoolean != 0:
+			return "Boolean"
+		case info&types.IsString != 0:
+			return "String"
+		default:
+			return "Any"
+		}
+	case *types.Map:
+		return "Map"
+	case *types.Struct:
+		return "Struct"
+	case *types.Slice:
+		if isByteArrayType(t) {
+			return "ByteArray"
+		}
+		return "Array"
+	default:
+		return "Any"
+	}
+}
+
+// MarshalJSON implements json.Marshaler interface.
+func (d *DebugRange) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + strconv.FormatUint(uint64(d.Start), 10) + `-` +
+		strconv.FormatUint(uint64(d.End), 10) + `"`), nil
+}
+
+// UnmarshalJSON implements json.Unmarshaler interface.
+func (d *DebugRange) UnmarshalJSON(data []byte) error {
+	startS, endS, err := parsePairJSON(data, "-")
+	if err != nil {
+		return err
+	}
+	start, err := strconv.ParseUint(startS, 10, 16)
+	if err != nil {
+		return err
+	}
+	end, err := strconv.ParseUint(endS, 10, 16)
+	if err != nil {
+		return err
+	}
+
+	d.Start = uint16(start)
+	d.End = uint16(end)
+
+	return nil
+}
+
+// MarshalJSON implements json.Marshaler interface.
+func (d *DebugParam) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + d.Name + `,` + d.Type + `"`), nil
+}
+
+// UnmarshalJSON implements json.Unmarshaler interface.
+func (d *DebugParam) UnmarshalJSON(data []byte) error {
+	startS, endS, err := parsePairJSON(data, ",")
+	if err != nil {
+		return err
+	}
+
+	d.Name = startS
+	d.Type = endS
+
+	return nil
+}
+
+// MarshalJSON implements json.Marshaler interface.
+func (d *DebugMethodName) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + d.Namespace + `,` + d.Name + `"`), nil
+}
+
+// UnmarshalJSON implements json.Unmarshaler interface.
+func (d *DebugMethodName) UnmarshalJSON(data []byte) error {
+	startS, endS, err := parsePairJSON(data, ",")
+	if err != nil {
+		return err
+	}
+
+	d.Namespace = startS
+	d.Name = endS
+
+	return nil
+}
+
+// MarshalJSON implements json.Marshaler interface.
+func (d *DebugSeqPoint) MarshalJSON() ([]byte, error) {
+	s := fmt.Sprintf("%d[%d]%d:%d-%d:%d", d.Opcode, d.Document,
+		d.StartLine, d.StartCol, d.EndLine, d.EndCol)
+	return []byte(`"` + s + `"`), nil
+}
+
+// UnmarshalJSON implements json.Unmarshaler interface.
+func (d *DebugSeqPoint) UnmarshalJSON(data []byte) error {
+	_, err := fmt.Sscanf(string(data), `"%d[%d]%d:%d-%d:%d"`,
+		&d.Opcode, &d.Document, &d.StartLine, &d.StartCol, &d.EndLine, &d.EndCol)
+	return err
+}
+
+func parsePairJSON(data []byte, sep string) (string, string, error) {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return "", "", err
+	}
+	ss := strings.SplitN(s, sep, 2)
+	if len(ss) != 2 {
+		return "", "", errors.New("invalid range format")
+	}
+	return ss[0], ss[1], nil
+}

pkg/compiler/debug_test.go

@@ -0,0 +1,141 @@
+package compiler
+
+import (
+	"testing"
+
+	"github.com/nspcc-dev/neo-go/pkg/internal/testserdes"
+	"github.com/nspcc-dev/neo-go/pkg/vm/opcode"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCodeGen_DebugInfo(t *testing.T) {
+	src := `package foo
+func Main(op string) bool {
+	var s string
+	_ = s
+	res := methodInt(op)
+	_ = methodString()	
+	_ = methodByteArray()
+	_ = methodArray()
+	_ = methodStruct()
+	return res == 42
+}
+
+func methodInt(a string) int {
+	if a == "get42" {
+		return 42
+	}
+	return 3
+}
+func methodString() string { return "" }
+func methodByteArray() []byte { return nil }
+func methodArray() []bool { return nil }
+func methodStruct() struct{} { return struct{}{} }
+`
+
+	info, err := getBuildInfo(src)
+	require.NoError(t, err)
+
+	pkg := info.program.Package(info.initialPackage)
+	c := newCodegen(info, pkg)
+	require.NoError(t, c.compile(info, pkg))
+
+	buf := c.prog.Bytes()
+	d := c.emitDebugInfo()
+	require.NotNil(t, d)
+
+	t.Run("return types", func(t *testing.T) {
+		returnTypes := map[string]string{
+			"methodInt":    "Integer",
+			"methodString": "String", "methodByteArray": "ByteArray",
+			"methodArray": "Array", "methodStruct": "Struct",
+			"Main": "Boolean",
+		}
+		for i := range d.Methods {
+			name := d.Methods[i].Name.Name
+			assert.Equal(t, returnTypes[name], d.Methods[i].ReturnType)
+		}
+	})
+
+	t.Run("variables", func(t *testing.T) {
+		vars := map[string][]string{
+			"Main": {"s,String", "res,Integer"},
+		}
+		for i := range d.Methods {
+			v, ok := vars[d.Methods[i].Name.Name]
+			if ok {
+				require.Equal(t, v, d.Methods[i].Variables)
+			}
+		}
+	})
+
+	// basic check that last instruction of every method is indeed RET
+	for i := range d.Methods {
+		index := d.Methods[i].Range.End
+		require.True(t, int(index) < len(buf))
+		require.EqualValues(t, opcode.RET, buf[index])
+	}
+}
+
+func TestSequencePoints(t *testing.T) {
+	src := `package foo
+	func Main(op string) bool {
+		if op == "123" {
+			return true
+		}
+		return false
+	}`
+
+	info, err := getBuildInfo(src)
+	require.NoError(t, err)
+
+	pkg := info.program.Package(info.initialPackage)
+	c := newCodegen(info, pkg)
+	require.NoError(t, c.compile(info, pkg))
+
+	d := c.emitDebugInfo()
+	require.NotNil(t, d)
+
+	// Main func has 2 return on 4-th and 6-th lines.
+	ps := d.Methods[0].SeqPoints
+	require.Equal(t, 2, len(ps))
+	require.Equal(t, 4, ps[0].StartLine)
+	require.Equal(t, 6, ps[1].StartLine)
+}
+
+func TestDebugInfo_MarshalJSON(t *testing.T) {
+	d := &DebugInfo{
+		EntryPoint: "main",
+		Documents:  []string{"/path/to/file"},
+		Methods: []MethodDebugInfo{
+			{
+				ID: "id1",
+				Name: DebugMethodName{
+					Namespace: "default",
+					Name:      "method1",
+				},
+				Range: DebugRange{Start: 10, End: 20},
+				Parameters: []DebugParam{
+					{"param1", "Integer"},
+					{"ok", "Boolean"},
+				},
+				ReturnType: "ByteArray",
+				Variables:  []string{},
+				SeqPoints: []DebugSeqPoint{
+					{
+						Opcode:    123,
+						Document:  1,
+						StartLine: 2,
+						StartCol:  3,
+						EndLine:   4,
+						EndCol:    5,
+					},
+				},
+			},
+		},
+		Events: []EventDebugInfo{},
+	}
+
+	testserdes.MarshalUnmarshalJSON(t, d, new(DebugInfo))
+}

pkg/compiler/func_scope.go

@@ -21,6 +21,11 @@ type funcScope struct {
 	// Program label of the scope
 	label uint16
 
+	// Range of opcodes corresponding to the function.
+	rng DebugRange
+	// Variables together with it's type in neo-vm.
+	variables []string
+
 	// Local variables
 	locals map[string]int
 
@@ -43,6 +48,7 @@ func newFuncScope(decl *ast.FuncDecl, label uint16) *funcScope {
 		label:     label,
 		locals:    map[string]int{},
 		voidCalls: map[*ast.CallExpr]bool{},
+		variables: []string{},
 		i:         -1,
 	}
 }