From 057e1c6e3c2c3ab087b4a6ee39395944e254eb36 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 10 Aug 2020 13:06:06 +0300 Subject: [PATCH 1/6] compiler: provide filename to `Compile()` --- cli/smartcontract/smart_contract.go | 2 +- pkg/compiler/compiler.go | 14 +++++++------- pkg/compiler/compiler_test.go | 2 +- pkg/compiler/debug_test.go | 4 ++-- pkg/compiler/global_test.go | 2 +- pkg/compiler/interop_test.go | 4 ++-- pkg/compiler/vm_test.go | 2 +- pkg/core/helper_test.go | 5 +++-- pkg/vm/cli/cli.go | 2 +- 9 files changed, 19 insertions(+), 18 deletions(-) diff --git a/cli/smartcontract/smart_contract.go b/cli/smartcontract/smart_contract.go index a6f62960e..4e0728354 100644 --- a/cli/smartcontract/smart_contract.go +++ b/cli/smartcontract/smart_contract.go @@ -568,7 +568,7 @@ func inspect(ctx *cli.Context) error { return cli.NewExitError(err, 1) } if compile { - b, err = compiler.Compile(bytes.NewReader(b)) + b, err = compiler.Compile(in, bytes.NewReader(b)) if err != nil { return cli.NewExitError(fmt.Errorf("failed to compile: %w", err), 1) } diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index b221b8d74..db3164053 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -84,9 +84,9 @@ func (c *codegen) fillImportMap(f *ast.File, pkg *types.Package) { } } -func getBuildInfo(src interface{}) (*buildInfo, error) { +func getBuildInfo(name string, src interface{}) (*buildInfo, error) { conf := loader.Config{ParserMode: parser.ParseComments} - f, err := conf.ParseFile("", src) + f, err := conf.ParseFile(name, src) if err != nil { return nil, err } @@ -104,8 +104,8 @@ func getBuildInfo(src interface{}) (*buildInfo, error) { } // Compile compiles a Go program into bytecode that can run on the NEO virtual machine. -func Compile(r io.Reader) ([]byte, error) { - buf, _, err := CompileWithDebugInfo(r) +func Compile(name string, r io.Reader) ([]byte, error) { + buf, _, err := CompileWithDebugInfo(name, r) if err != nil { return nil, err } @@ -114,8 +114,8 @@ func Compile(r io.Reader) ([]byte, error) { } // CompileWithDebugInfo compiles a Go program into bytecode and emits debug info. -func CompileWithDebugInfo(r io.Reader) ([]byte, *DebugInfo, error) { - ctx, err := getBuildInfo(r) +func CompileWithDebugInfo(name string, r io.Reader) ([]byte, *DebugInfo, error) { + ctx, err := getBuildInfo(name, r) if err != nil { return nil, nil, err } @@ -138,7 +138,7 @@ func CompileAndSave(src string, o *Options) ([]byte, error) { if err != nil { return nil, err } - b, di, err := CompileWithDebugInfo(bytes.NewReader(b)) + b, di, err := CompileWithDebugInfo(src, bytes.NewReader(b)) if err != nil { return nil, fmt.Errorf("error while trying to compile smart contract file: %w", err) } diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go index d3ab69f1b..2e114cbef 100644 --- a/pkg/compiler/compiler_test.go +++ b/pkg/compiler/compiler_test.go @@ -77,6 +77,6 @@ func compileFile(src string) error { if err != nil { return err } - _, err = compiler.Compile(file) + _, err = compiler.Compile("foo.go", file) return err } diff --git a/pkg/compiler/debug_test.go b/pkg/compiler/debug_test.go index 6c5a415cd..3134d4f17 100644 --- a/pkg/compiler/debug_test.go +++ b/pkg/compiler/debug_test.go @@ -44,7 +44,7 @@ func MethodStruct() struct{} { return struct{}{} } func unexportedMethod() int { return 1 } ` - info, err := getBuildInfo(src) + info, err := getBuildInfo("foo.go", src) require.NoError(t, err) pkg := info.program.Package(info.initialPackage) @@ -238,7 +238,7 @@ func TestSequencePoints(t *testing.T) { return false }` - info, err := getBuildInfo(src) + info, err := getBuildInfo("foo.go", src) require.NoError(t, err) pkg := info.program.Package(info.initialPackage) diff --git a/pkg/compiler/global_test.go b/pkg/compiler/global_test.go index 934972fec..2be4caf30 100644 --- a/pkg/compiler/global_test.go +++ b/pkg/compiler/global_test.go @@ -118,7 +118,7 @@ func TestContractWithNoMain(t *testing.T) { someLocal := 2 return someGlobal + someLocal + a }` - b, di, err := compiler.CompileWithDebugInfo(strings.NewReader(src)) + b, di, err := compiler.CompileWithDebugInfo("foo.go", strings.NewReader(src)) require.NoError(t, err) v := vm.New() invokeMethod(t, "Add3", b, v, di) diff --git a/pkg/compiler/interop_test.go b/pkg/compiler/interop_test.go index e0cceeb59..79c00d93f 100644 --- a/pkg/compiler/interop_test.go +++ b/pkg/compiler/interop_test.go @@ -63,7 +63,7 @@ func TestFromAddress(t *testing.T) { } func spawnVM(t *testing.T, ic *interop.Context, src string) *vm.VM { - b, di, err := compiler.CompileWithDebugInfo(strings.NewReader(src)) + b, di, err := compiler.CompileWithDebugInfo("foo.go", strings.NewReader(src)) require.NoError(t, err) v := core.SpawnVM(ic) invokeMethod(t, testMainIdent, b, v, di) @@ -86,7 +86,7 @@ func TestAppCall(t *testing.T) { } ` - inner, di, err := compiler.CompileWithDebugInfo(strings.NewReader(srcInner)) + inner, di, err := compiler.CompileWithDebugInfo("foo.go", strings.NewReader(srcInner)) require.NoError(t, err) m, err := di.ConvertToManifest(smartcontract.NoProperties) require.NoError(t, err) diff --git a/pkg/compiler/vm_test.go b/pkg/compiler/vm_test.go index ce862a93e..d4a317d36 100644 --- a/pkg/compiler/vm_test.go +++ b/pkg/compiler/vm_test.go @@ -72,7 +72,7 @@ func vmAndCompileInterop(t *testing.T, src string) (*vm.VM, *storagePlugin) { vm.GasLimit = -1 vm.SyscallHandler = storePlugin.syscallHandler - b, di, err := compiler.CompileWithDebugInfo(strings.NewReader(src)) + b, di, err := compiler.CompileWithDebugInfo("foo.go", strings.NewReader(src)) require.NoError(t, err) invokeMethod(t, testMainIdent, b, vm, di) diff --git a/pkg/core/helper_test.go b/pkg/core/helper_test.go index 9b6e2f9fa..fb4edb92e 100644 --- a/pkg/core/helper_test.go +++ b/pkg/core/helper_test.go @@ -225,9 +225,10 @@ func TestCreateBasicChain(t *testing.T) { require.NoError(t, err) // Push some contract into the chain. - c, err := ioutil.ReadFile(prefix + "test_contract.go") + name := prefix + "test_contract.go" + c, err := ioutil.ReadFile(name) require.NoError(t, err) - avm, di, err := compiler.CompileWithDebugInfo(bytes.NewReader(c)) + avm, di, err := compiler.CompileWithDebugInfo(name, bytes.NewReader(c)) require.NoError(t, err) t.Logf("contractHash: %s", hash.Hash160(avm).StringLE()) diff --git a/pkg/vm/cli/cli.go b/pkg/vm/cli/cli.go index c2109c6c6..cc1011cb8 100644 --- a/pkg/vm/cli/cli.go +++ b/pkg/vm/cli/cli.go @@ -311,7 +311,7 @@ func handleLoadGo(c *ishell.Context) { c.Err(err) return } - b, err := compiler.Compile(bytes.NewReader(fb)) + b, err := compiler.Compile(c.Args[0], bytes.NewReader(fb)) if err != nil { c.Err(err) return From 40fa7c0f6e52eb42c971ab1f66d20a2297ab309d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 10 Aug 2020 13:10:35 +0300 Subject: [PATCH 2/6] compiler: emit all used files in DebugInfo.Documents --- pkg/compiler/analysis.go | 11 +++++++++++ pkg/compiler/codegen.go | 7 +++++++ pkg/compiler/compiler.go | 7 ------- pkg/compiler/debug.go | 8 +++++--- pkg/compiler/debug_test.go | 2 ++ 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/pkg/compiler/analysis.go b/pkg/compiler/analysis.go index 899d9e35f..aad9ea6a7 100644 --- a/pkg/compiler/analysis.go +++ b/pkg/compiler/analysis.go @@ -3,6 +3,7 @@ package compiler import ( "errors" "go/ast" + "go/token" "go/types" "strings" @@ -163,6 +164,16 @@ func (c *codegen) visitPkg(pkg *types.Package, seen map[string]bool) { c.packages = append(c.packages, pkgPath) } +func (c *codegen) fillDocumentInfo() { + fset := c.buildInfo.program.Fset + fset.Iterate(func(f *token.File) bool { + filePath := f.Position(f.Pos(0)).Filename + c.docIndex[filePath] = len(c.documents) + c.documents = append(c.documents, filePath) + return true + }) +} + func (c *codegen) analyzeFuncUsage() funcUsage { usage := funcUsage{} diff --git a/pkg/compiler/codegen.go b/pkg/compiler/codegen.go index 1f6f3bff4..453d55160 100644 --- a/pkg/compiler/codegen.go +++ b/pkg/compiler/codegen.go @@ -77,6 +77,11 @@ type codegen struct { // packages contains packages in the order they were loaded. packages []string + // documents contains paths to all files used by the program. + documents []string + // docIndex maps file path to an index in documents array. + docIndex map[string]int + // Label table for recording jump destinations. l []int } @@ -1541,6 +1546,7 @@ func (c *codegen) newLambda(u uint16, lit *ast.FuncLit) { func (c *codegen) compile(info *buildInfo, pkg *loader.PackageInfo) error { c.mainPkg = pkg c.analyzePkgOrder() + c.fillDocumentInfo() funUsage := c.analyzeFuncUsage() // Bring all imported functions into scope. @@ -1588,6 +1594,7 @@ func newCodegen(info *buildInfo, pkg *loader.PackageInfo) *codegen { labels: map[labelWithType]uint16{}, typeInfo: &pkg.Info, constMap: map[string]types.TypeAndValue{}, + docIndex: map[string]int{}, sequencePoints: make(map[string][]DebugSeqPoint), } diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index db3164053..740c990bd 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -10,7 +10,6 @@ import ( "io" "io/ioutil" "os" - "path/filepath" "strings" "github.com/nspcc-dev/neo-go/pkg/smartcontract" @@ -159,12 +158,6 @@ func CompileAndSave(src string, o *Options) ([]byte, error) { return b, nil } - p, err := filepath.Abs(src) - if err != nil { - return b, err - } - di.Documents = append(di.Documents, p) - if o.DebugInfo != "" { data, err := json.Marshal(di) if err != nil { diff --git a/pkg/compiler/debug.go b/pkg/compiler/debug.go index 2edfc3150..7b297298d 100644 --- a/pkg/compiler/debug.go +++ b/pkg/compiler/debug.go @@ -94,6 +94,7 @@ func (c *codegen) saveSequencePoint(n ast.Node) { end := fset.Position(n.End()) c.sequencePoints[c.scope.name] = append(c.sequencePoints[c.scope.name], DebugSeqPoint{ Opcode: c.prog.Len(), + Document: c.docIndex[start.Filename], StartLine: start.Line, StartCol: start.Offset, EndLine: end.Line, @@ -103,9 +104,10 @@ func (c *codegen) saveSequencePoint(n ast.Node) { func (c *codegen) emitDebugInfo(contract []byte) *DebugInfo { d := &DebugInfo{ - MainPkg: c.mainPkg.Pkg.Name(), - Hash: hash.Hash160(contract), - Events: []EventDebugInfo{}, + MainPkg: c.mainPkg.Pkg.Name(), + Hash: hash.Hash160(contract), + Events: []EventDebugInfo{}, + Documents: c.documents, } if c.initEndOffset > 0 { d.Methods = append(d.Methods, MethodDebugInfo{ diff --git a/pkg/compiler/debug_test.go b/pkg/compiler/debug_test.go index 3134d4f17..fee427df5 100644 --- a/pkg/compiler/debug_test.go +++ b/pkg/compiler/debug_test.go @@ -249,6 +249,8 @@ func TestSequencePoints(t *testing.T) { d := c.emitDebugInfo(buf) require.NotNil(t, d) + require.Equal(t, d.Documents, []string{"foo.go"}) + // Main func has 2 return on 4-th and 6-th lines. ps := d.Methods[0].SeqPoints require.Equal(t, 2, len(ps)) From 128626de5cbe45d94f6bb2489c82bd48c6b051b3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 10 Aug 2020 17:42:53 +0300 Subject: [PATCH 3/6] compiler: save sequence points for `init` function --- pkg/compiler/debug.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/compiler/debug.go b/pkg/compiler/debug.go index 7b297298d..5aad14726 100644 --- a/pkg/compiler/debug.go +++ b/pkg/compiler/debug.go @@ -85,14 +85,15 @@ type DebugParam struct { } func (c *codegen) saveSequencePoint(n ast.Node) { - if c.scope == nil { - // do not save globals for now - return + name := "init" + if c.scope != nil { + name = c.scope.name } + fset := c.buildInfo.program.Fset start := fset.Position(n.Pos()) end := fset.Position(n.End()) - c.sequencePoints[c.scope.name] = append(c.sequencePoints[c.scope.name], DebugSeqPoint{ + c.sequencePoints[name] = append(c.sequencePoints[name], DebugSeqPoint{ Opcode: c.prog.Len(), Document: c.docIndex[start.Filename], StartLine: start.Line, @@ -122,6 +123,7 @@ func (c *codegen) emitDebugInfo(contract []byte) *DebugInfo { End: uint16(c.initEndOffset), }, ReturnType: "Void", + SeqPoints: c.sequencePoints["init"], }) } for name, scope := range c.funcs { From 553e57c2c4c86714e0f4c7a6e29f61b304e75597 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 10 Aug 2020 17:43:26 +0300 Subject: [PATCH 4/6] compiler: make sequence points on global var/const declarations --- pkg/compiler/codegen.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/compiler/codegen.go b/pkg/compiler/codegen.go index 453d55160..45deb1d85 100644 --- a/pkg/compiler/codegen.go +++ b/pkg/compiler/codegen.go @@ -400,6 +400,9 @@ func (c *codegen) Visit(node ast.Node) ast.Visitor { // x = 2 // ) case *ast.GenDecl: + if n.Tok == token.VAR || n.Tok == token.CONST { + c.saveSequencePoint(n) + } if n.Tok == token.CONST { for _, spec := range n.Specs { vs := spec.(*ast.ValueSpec) From a34ba92d460f509860f7848455b2b6a32db5493e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 10 Aug 2020 18:23:45 +0300 Subject: [PATCH 5/6] compiler: allow to split main package across multiple files --- cli/smartcontract/smart_contract.go | 9 ++--- pkg/compiler/compiler.go | 51 ++++++++++++++++++++-------- pkg/compiler/compiler_test.go | 20 ++++++++--- pkg/compiler/testdata/multi/file1.go | 4 +++ pkg/compiler/testdata/multi/file2.go | 4 +++ pkg/vm/cli/cli.go | 8 +---- 6 files changed, 63 insertions(+), 33 deletions(-) diff --git a/cli/smartcontract/smart_contract.go b/cli/smartcontract/smart_contract.go index 4e0728354..ab8016d01 100644 --- a/cli/smartcontract/smart_contract.go +++ b/cli/smartcontract/smart_contract.go @@ -1,7 +1,6 @@ package smartcontract import ( - "bytes" "encoding/hex" "encoding/json" "errors" @@ -563,12 +562,10 @@ func inspect(ctx *cli.Context) error { if len(in) == 0 { return cli.NewExitError(errNoInput, 1) } - b, err := ioutil.ReadFile(in) - if err != nil { - return cli.NewExitError(err, 1) - } + var b []byte + var err error if compile { - b, err = compiler.Compile(in, bytes.NewReader(b)) + b, err = compiler.Compile(in, nil) if err != nil { return cli.NewExitError(fmt.Errorf("failed to compile: %w", err), 1) } diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 740c990bd..7c131319f 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -1,8 +1,8 @@ package compiler import ( - "bytes" "encoding/json" + "errors" "fmt" "go/ast" "go/parser" @@ -10,6 +10,7 @@ import ( "io" "io/ioutil" "os" + "path" "strings" "github.com/nspcc-dev/neo-go/pkg/smartcontract" @@ -85,11 +86,32 @@ func (c *codegen) fillImportMap(f *ast.File, pkg *types.Package) { func getBuildInfo(name string, src interface{}) (*buildInfo, error) { conf := loader.Config{ParserMode: parser.ParseComments} - f, err := conf.ParseFile(name, src) - if err != nil { - return nil, err + if src != nil { + f, err := conf.ParseFile(name, src) + if err != nil { + return nil, err + } + conf.CreateFromFiles("", f) + } else { + var names []string + if strings.HasSuffix(name, ".go") { + names = append(names, name) + } else { + ds, err := ioutil.ReadDir(name) + if err != nil { + return nil, fmt.Errorf("'%s' is neither Go source nor a directory", name) + } + for i := range ds { + if !ds[i].IsDir() && strings.HasSuffix(ds[i].Name(), ".go") { + names = append(names, path.Join(name, ds[i].Name())) + } + } + } + if len(names) == 0 { + return nil, errors.New("no files provided") + } + conf.CreateFromFilenames("", names...) } - conf.CreateFromFiles("", f) prog, err := conf.Load() if err != nil { @@ -97,12 +119,14 @@ func getBuildInfo(name string, src interface{}) (*buildInfo, error) { } return &buildInfo{ - initialPackage: f.Name.Name, + initialPackage: prog.InitialPackages()[0].Pkg.Name(), program: prog, }, nil } // Compile compiles a Go program into bytecode that can run on the NEO virtual machine. +// If `r != nil`, `name` is interpreted as a filename, and `r` as file contents. +// Otherwise `name` is either file name or name of the directory containing source files. func Compile(name string, r io.Reader) ([]byte, error) { buf, _, err := CompileWithDebugInfo(name, r) if err != nil { @@ -123,21 +147,18 @@ func CompileWithDebugInfo(name string, r io.Reader) ([]byte, *DebugInfo, error) // CompileAndSave will compile and save the file to disk in the NEF format. func CompileAndSave(src string, o *Options) ([]byte, error) { - if !strings.HasSuffix(src, ".go") { - return nil, fmt.Errorf("%s is not a Go file", src) - } o.Outfile = strings.TrimSuffix(o.Outfile, fmt.Sprintf(".%s", fileExt)) if len(o.Outfile) == 0 { - o.Outfile = strings.TrimSuffix(src, ".go") + if strings.HasSuffix(src, ".go") { + o.Outfile = strings.TrimSuffix(src, ".go") + } else { + o.Outfile = "out" + } } if len(o.Ext) == 0 { o.Ext = fileExt } - b, err := ioutil.ReadFile(src) - if err != nil { - return nil, err - } - b, di, err := CompileWithDebugInfo(src, bytes.NewReader(b)) + b, di, err := CompileWithDebugInfo(src, nil) if err != nil { return nil, fmt.Errorf("error while trying to compile smart contract file: %w", err) } diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go index 2e114cbef..797e0a16a 100644 --- a/pkg/compiler/compiler_test.go +++ b/pkg/compiler/compiler_test.go @@ -24,6 +24,20 @@ func TestCompiler(t *testing.T) { // CompileAndSave use config.Version for proper .nef generation. config.Version = "0.90.0-test" testCases := []compilerTestCase{ + { + name: "TestCompileDirectory", + function: func(t *testing.T) { + const multiMainDir = "testdata/multi" + _, di, err := compiler.CompileWithDebugInfo(multiMainDir, nil) + require.NoError(t, err) + m := map[string]bool{} + for i := range di.Methods { + m[di.Methods[i].Name.Name] = true + } + require.Contains(t, m, "Func1") + require.Contains(t, m, "Func2") + }, + }, { name: "TestCompile", function: func(t *testing.T) { @@ -73,10 +87,6 @@ func filterFilename(infos []os.FileInfo) string { } func compileFile(src string) error { - file, err := os.Open(src) - if err != nil { - return err - } - _, err = compiler.Compile("foo.go", file) + _, err := compiler.Compile(src, nil) return err } diff --git a/pkg/compiler/testdata/multi/file1.go b/pkg/compiler/testdata/multi/file1.go index c51714e74..9cbe0cd1d 100644 --- a/pkg/compiler/testdata/multi/file1.go +++ b/pkg/compiler/testdata/multi/file1.go @@ -3,3 +3,7 @@ package multi var SomeVar12 = 12 const SomeConst = 42 + +func Func1() bool { + return true +} diff --git a/pkg/compiler/testdata/multi/file2.go b/pkg/compiler/testdata/multi/file2.go index 2ee034599..a96ba78d9 100644 --- a/pkg/compiler/testdata/multi/file2.go +++ b/pkg/compiler/testdata/multi/file2.go @@ -5,3 +5,7 @@ var SomeVar30 = 30 func Sum() int { return SomeVar12 + SomeVar30 } + +func Func2() bool { + return false +} diff --git a/pkg/vm/cli/cli.go b/pkg/vm/cli/cli.go index cc1011cb8..a5fc1b060 100644 --- a/pkg/vm/cli/cli.go +++ b/pkg/vm/cli/cli.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "errors" "fmt" - "io/ioutil" "math/big" "os" "strconv" @@ -306,12 +305,7 @@ func handleLoadGo(c *ishell.Context) { c.Err(errors.New("missing parameter ")) return } - fb, err := ioutil.ReadFile(c.Args[0]) - if err != nil { - c.Err(err) - return - } - b, err := compiler.Compile(c.Args[0], bytes.NewReader(fb)) + b, err := compiler.Compile(c.Args[0], nil) if err != nil { c.Err(err) return From 6d169a356e36d1dbc4f5b359519e889524ee58da Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Aug 2020 11:20:47 +0300 Subject: [PATCH 6/6] docs: update compiler.md Make a note about compiling directories. --- docs/compiler.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/compiler.md b/docs/compiler.md index 6aeaf541d..8b2613478 100644 --- a/docs/compiler.md +++ b/docs/compiler.md @@ -45,6 +45,12 @@ By default the filename will be the name of your .go file with the .nef extensio ./bin/neo-go contract compile -i mycontract.go --out /Users/foo/bar/contract.nef ``` +If you contract is split across multiple files, you must provide a path +to the directory where package files are contained instead of a single Go file: +``` +./bin/neo-go contract compile -i ./path/to/contract +``` + ### Debugging You can dump the opcodes generated by the compiler with the following command: