neotest: implement coverage collection

Test coverage is automatically enabled when go test is running with coverage
enabled. It can be disabled for any Executor by using relevant methods.
Coverage is gathered by capturing VM OPs during test contract execution and
mapping them to the contract source code using the DebugInfo information.

Signed-off-by: Slava0135 <super.novalskiy_0135@inbox.ru>

.github/workflows/tests.yml

@@ -139,7 +139,7 @@ jobs:
           cache: true
 
       - name: Write coverage profile
-        run: go test -timeout 15m -v ./... -coverprofile=./coverage.txt -covermode=atomic -coverpkg=./pkg...,./cli/...
+        run: DISABLE_NEOTEST_COVER=1 go test -timeout 15m -v ./... -coverprofile=./coverage.txt -covermode=atomic -coverpkg=./pkg...,./cli/...
 
       - name: Upload coverage results to Codecov
         uses: codecov/codecov-action@v4

Makefile

@@ -22,6 +22,8 @@ BUILD_FLAGS = "-X '$(REPO)/pkg/config.Version=$(VERSION)' -X '$(REPO)/cli/smartc
 
 IMAGE_REPO=nspccdev/neo-go
 
+DISABLE_NEOTEST_COVER=1
+
 # All of the targets are phony here because we don't really use make dependency
 # tracking for files
 .PHONY: build $(BINARY) deps image docker/$(BINARY) image-latest image-push image-push-latest clean-cluster \

pkg/neotest/basic.go

@@ -32,20 +32,23 @@ type Executor struct {
 	Validator     Signer
 	Committee     Signer
 	CommitteeHash util.Uint160
-	Contracts     map[string]*Contract
+	// collectCoverage is true if coverage is being collected when running this executor.
+	collectCoverage bool
 }
 
 // NewExecutor creates a new executor instance from the provided blockchain and committee.
+// By default coverage collection is enabled, but only when `go test` is running with coverage enabled.
+// Use DisableCoverage and EnableCoverage to stop coverage collection for this executor when not desired.
 func NewExecutor(t testing.TB, bc *core.Blockchain, validator, committee Signer) *Executor {
 	checkMultiSigner(t, validator)
 	checkMultiSigner(t, committee)
 
 	return &Executor{
-		Chain:         bc,
-		Validator:     validator,
-		Committee:     committee,
-		CommitteeHash: committee.ScriptHash(),
-		Contracts:     make(map[string]*Contract),
+		Chain:           bc,
+		Validator:       validator,
+		Committee:       committee,
+		CommitteeHash:   committee.ScriptHash(),
+		collectCoverage: isCoverageEnabled(),
 	}
 }
 
@@ -145,6 +148,7 @@ func (e *Executor) DeployContract(t testing.TB, c *Contract, data any) util.Uint
 // data is an optional argument to `_deploy`.
 // It returns the hash of the deploy transaction.
 func (e *Executor) DeployContractBy(t testing.TB, signer Signer, c *Contract, data any) util.Uint256 {
+	e.trackCoverage(t, c)
 	tx := e.NewDeployTxBy(t, signer, c, data)
 	e.AddNewBlock(t, tx)
 	e.CheckHalt(t, tx.Hash())
@@ -164,11 +168,22 @@ func (e *Executor) DeployContractBy(t testing.TB, signer Signer, c *Contract, da
 // DeployContractCheckFAULT compiles and deploys a contract to the bc using the validator
 // account. It checks that the deploy transaction FAULTed with the specified error.
 func (e *Executor) DeployContractCheckFAULT(t testing.TB, c *Contract, data any, errMessage string) {
+	e.trackCoverage(t, c)
 	tx := e.NewDeployTx(t, c, data)
 	e.AddNewBlock(t, tx)
 	e.CheckFault(t, tx.Hash(), errMessage)
 }
 
+// trackCoverage switches on coverage tracking for provided script if `go test` is running with coverage enabled.
+func (e *Executor) trackCoverage(t testing.TB, c *Contract) {
+	if e.collectCoverage {
+		addScriptToCoverage(c)
+		t.Cleanup(func() {
+			reportCoverage(t)
+		})
+	}
+}
+
 // InvokeScript adds a transaction with the specified script to the chain and
 // returns its hash. It does no faults check.
 func (e *Executor) InvokeScript(t testing.TB, script []byte, signers []Signer) util.Uint256 {
@@ -401,6 +416,10 @@ func (e *Executor) TestInvoke(tx *transaction.Transaction) (*vm.VM, error) {
 	ttx := *tx
 	ic, _ := e.Chain.GetTestVM(trigger.Application, &ttx, b)
 
+	if e.collectCoverage {
+		ic.VM.SetOnExecHook(coverageHook)
+	}
+
 	defer ic.Finalize()
 
 	ic.VM.LoadWithFlags(tx.Script, callflag.All)
@@ -431,3 +450,13 @@ func (e *Executor) GetTxExecResult(t testing.TB, h util.Uint256) *state.AppExecR
 	require.Equal(t, 1, len(aer))
 	return &aer[0]
 }
+
+// EnableCoverage enables coverage collection for this executor, but only when `go test` is running with coverage enabled.
+func (e *Executor) EnableCoverage() {
+	e.collectCoverage = isCoverageEnabled()
+}
+
+// DisableCoverage disables coverage collection for this executor until enabled explicitly through EnableCoverage.
+func (e *Executor) DisableCoverage() {
+	e.collectCoverage = false
+}

pkg/neotest/client.go

@@ -63,6 +63,10 @@ func (c *ContractInvoker) TestInvokeScript(t testing.TB, script []byte, signers
 	}
 	t.Cleanup(ic.Finalize)
 
+	if c.collectCoverage {
+		ic.VM.SetOnExecHook(coverageHook)
+	}
+
 	ic.VM.LoadWithFlags(tx.Script, callflag.All)
 	err = ic.VM.Run()
 	return ic.VM.Estack(), err
@@ -78,6 +82,10 @@ func (c *ContractInvoker) TestInvoke(t testing.TB, method string, args ...any) (
 	}
 	t.Cleanup(ic.Finalize)
 
+	if c.collectCoverage {
+		ic.VM.SetOnExecHook(coverageHook)
+	}
+
 	ic.VM.LoadWithFlags(tx.Script, callflag.All)
 	err = ic.VM.Run()
 	return ic.VM.Estack(), err

pkg/neotest/compile.go

@@ -16,9 +16,10 @@ import (
 
 // Contract contains contract info for deployment.
 type Contract struct {
-	Hash     util.Uint160
-	NEF      *nef.File
-	Manifest *manifest.Manifest
+	Hash      util.Uint160
+	NEF       *nef.File
+	Manifest  *manifest.Manifest
+	DebugInfo *compiler.DebugInfo
 }
 
 // contracts caches the compiled contracts from FS across multiple tests.
@@ -36,9 +37,10 @@ func CompileSource(t testing.TB, sender util.Uint160, src io.Reader, opts *compi
 	require.NoError(t, err)
 
 	return &Contract{
-		Hash:     state.CreateContractHash(sender, ne.Checksum, m.Name),
-		NEF:      ne,
-		Manifest: m,
+		Hash:      state.CreateContractHash(sender, ne.Checksum, m.Name),
+		NEF:       ne,
+		Manifest:  m,
+		DebugInfo: di,
 	}
 }
 
@@ -73,9 +75,10 @@ func CompileFile(t testing.TB, sender util.Uint160, srcPath string, configPath s
 	require.NoError(t, err)
 
 	c := &Contract{
-		Hash:     state.CreateContractHash(sender, ne.Checksum, m.Name),
-		NEF:      ne,
-		Manifest: m,
+		Hash:      state.CreateContractHash(sender, ne.Checksum, m.Name),
+		NEF:       ne,
+		Manifest:  m,
+		DebugInfo: di,
 	}
 	contracts[srcPath] = c
 	return c

pkg/neotest/coverage.go

@@ -0,0 +1,211 @@
+package neotest
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/nspcc-dev/neo-go/pkg/compiler"
+	"github.com/nspcc-dev/neo-go/pkg/util"
+	"github.com/nspcc-dev/neo-go/pkg/vm"
+	"github.com/nspcc-dev/neo-go/pkg/vm/opcode"
+)
+
+const (
+	// goCoverProfileFlag specifies the name of `go test` flag that tells it where to save coverage data.
+	// Neotest coverage can be enabled only when this flag is set.
+	goCoverProfileFlag = "test.coverprofile"
+	// disableNeotestCover is name of the environmental variable used to explicitly disable neotest coverage.
+	disableNeotestCover = "DISABLE_NEOTEST_COVER"
+)
+
+var (
+	// coverageLock protects all vars below from concurrent modification when tests are run in parallel.
+	coverageLock sync.Mutex
+	// rawCoverage maps script hash to coverage data, collected during testing.
+	rawCoverage = make(map[util.Uint160]*scriptRawCoverage)
+	// flagChecked is true if `go test` coverage flag was checked at any point.
+	flagChecked bool
+	// coverageEnabled is true if coverage is being collected on test execution.
+	coverageEnabled bool
+	// coverProfile specifies the file all coverage data is written to, unless empty.
+	coverProfile = ""
+)
+
+type scriptRawCoverage struct {
+	debugInfo      *compiler.DebugInfo
+	offsetsVisited []int
+}
+
+type coverBlock struct {
+	// Line number for block start.
+	startLine uint
+	// Column number for block start.
+	startCol uint
+	// Line number for block end.
+	endLine uint
+	// Column number for block end.
+	endCol uint
+	// Number of statements included in this block.
+	stmts uint
+	// Number of times this block was executed.
+	counts uint
+}
+
+// documentName makes it clear when a `string` maps path to the script file.
+type documentName = string
+
+func isCoverageEnabled() bool {
+	coverageLock.Lock()
+	defer coverageLock.Unlock()
+
+	if flagChecked {
+		return coverageEnabled
+	}
+	defer func() { flagChecked = true }()
+
+	var disabledByEnvironment bool
+	if v, ok := os.LookupEnv(disableNeotestCover); ok {
+		disabled, err := strconv.ParseBool(v)
+		if err != nil {
+			panic(fmt.Sprintf("coverage: error when parsing environment variable '%s', expected bool, but got '%s'", disableNeotestCover, v))
+		}
+		disabledByEnvironment = disabled
+	}
+
+	var goToolCoverageEnabled bool
+	flag.VisitAll(func(f *flag.Flag) {
+		if f.Name == goCoverProfileFlag && f.Value != nil && f.Value.String() != "" {
+			goToolCoverageEnabled = true
+			coverProfile = f.Value.String()
+		}
+	})
+
+	coverageEnabled = !disabledByEnvironment && goToolCoverageEnabled
+
+	if coverageEnabled {
+		// This is needed so go cover tool doesn't overwrite
+		// the file with our coverage when all tests are done.
+		err := flag.Set(goCoverProfileFlag, "")
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	return coverageEnabled
+}
+
+var coverageHook vm.OnExecHook = func(scriptHash util.Uint160, offset int, opcode opcode.Opcode) {
+	coverageLock.Lock()
+	defer coverageLock.Unlock()
+	if cov, ok := rawCoverage[scriptHash]; ok {
+		cov.offsetsVisited = append(cov.offsetsVisited, offset)
+	}
+}
+
+func reportCoverage(t testing.TB) {
+	coverageLock.Lock()
+	defer coverageLock.Unlock()
+	f, err := os.Create(coverProfile)
+	if err != nil {
+		t.Fatalf("coverage: can't create file '%s' to write coverage report", coverProfile)
+	}
+	defer f.Close()
+	writeCoverageReport(f)
+}
+
+func writeCoverageReport(w io.Writer) {
+	fmt.Fprintf(w, "mode: set\n")
+	cover := processCover()
+	for name, blocks := range cover {
+		for _, b := range blocks {
+			c := 0
+			if b.counts > 0 {
+				c = 1
+			}
+			fmt.Fprintf(w, "%s:%d.%d,%d.%d %d %d\n", name,
+				b.startLine, b.startCol,
+				b.endLine, b.endCol,
+				b.stmts,
+				c,
+			)
+		}
+	}
+}
+
+func processCover() map[documentName][]coverBlock {
+	documents := make(map[documentName]struct{})
+	for _, scriptRawCoverage := range rawCoverage {
+		for _, documentName := range scriptRawCoverage.debugInfo.Documents {
+			documents[documentName] = struct{}{}
+		}
+	}
+
+	cover := make(map[documentName][]coverBlock)
+
+	for documentName := range documents {
+		mappedBlocks := make(map[int]*coverBlock)
+
+		for _, scriptRawCoverage := range rawCoverage {
+			di := scriptRawCoverage.debugInfo
+			documentSeqPoints := documentSeqPoints(di, documentName)
+
+			for _, point := range documentSeqPoints {
+				b := coverBlock{
+					startLine: uint(point.StartLine),
+					startCol:  uint(point.StartCol),
+					endLine:   uint(point.EndLine),
+					endCol:    uint(point.EndCol),
+					stmts:     1 + uint(point.EndLine) - uint(point.StartLine),
+					counts:    0,
+				}
+				mappedBlocks[point.Opcode] = &b
+			}
+		}
+
+		for _, scriptRawCoverage := range rawCoverage {
+			di := scriptRawCoverage.debugInfo
+			documentSeqPoints := documentSeqPoints(di, documentName)
+
+			for _, offset := range scriptRawCoverage.offsetsVisited {
+				for _, point := range documentSeqPoints {
+					if point.Opcode == offset {
+						mappedBlocks[point.Opcode].counts++
+					}
+				}
+			}
+		}
+
+		var blocks []coverBlock
+		for _, b := range mappedBlocks {
+			blocks = append(blocks, *b)
+		}
+		cover[documentName] = blocks
+	}
+
+	return cover
+}
+
+func documentSeqPoints(di *compiler.DebugInfo, doc documentName) []compiler.DebugSeqPoint {
+	var res []compiler.DebugSeqPoint
+	for _, methodDebugInfo := range di.Methods {
+		for _, p := range methodDebugInfo.SeqPoints {
+			if di.Documents[p.Document] == doc {
+				res = append(res, p)
+			}
+		}
+	}
+	return res
+}
+
+func addScriptToCoverage(c *Contract) {
+	coverageLock.Lock()
+	defer coverageLock.Unlock()
+	if _, ok := rawCoverage[c.Hash]; !ok {
+		rawCoverage[c.Hash] = &scriptRawCoverage{debugInfo: c.DebugInfo}
+	}
+}

pkg/neotest/doc.go

@@ -21,5 +21,13 @@ them in the same package with the smart contract iself can lead to unxpected
 results if smart contract has any init() functions. If that's the case they
 will be compiled into the testing binary even when using package_test and their
 execution can affect tests. See https://github.com/nspcc-dev/neo-go/issues/3120 for details.
+
+Test coverage for contracts is automatically enabled when `go test` is running with
+coverage enabled. When not desired, it can be disabled for any Executor by using
+EnableCoverage and DisableCoverage. Be aware that coverage data collected by `go test`
+itself will not be saved because it will be replaced with contracts coverage instead.
+In case `go test` coverage is wanted DISABLE_NEOTEST_COVER=1 variable can be set.
+Coverage is gathered by capturing VM instructions during test contract execution and
+mapping them to the contract source code using the DebugInfo information.
 */
 package neotest