From d0c45477f5da4363661c5a797819619189b8da78 Mon Sep 17 00:00:00 2001 From: Slava0135 Date: Mon, 19 Aug 2024 14:39:18 +0300 Subject: [PATCH] neotest: implement coverage collection Test coverage is automatically enabled when go test is running with coverage enabled. It can be disabled for any Executor by using relevant methods. Coverage is gathered by capturing VM OPs during test contract execution and mapping them to the contract source code using the DebugInfo information. Signed-off-by: Slava0135 --- .github/workflows/tests.yml | 2 +- Makefile | 2 + pkg/neotest/basic.go | 41 ++++++- pkg/neotest/client.go | 8 ++ pkg/neotest/compile.go | 21 ++-- pkg/neotest/coverage.go | 211 ++++++++++++++++++++++++++++++++++++ pkg/neotest/doc.go | 8 ++ 7 files changed, 277 insertions(+), 16 deletions(-) create mode 100644 pkg/neotest/coverage.go diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 524214bd1..ff74ab05d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -139,7 +139,7 @@ jobs: cache: true - name: Write coverage profile - run: go test -timeout 15m -v ./... -coverprofile=./coverage.txt -covermode=atomic -coverpkg=./pkg...,./cli/... + run: DISABLE_NEOTEST_COVER=1 go test -timeout 15m -v ./... -coverprofile=./coverage.txt -covermode=atomic -coverpkg=./pkg...,./cli/... - name: Upload coverage results to Codecov uses: codecov/codecov-action@v4 diff --git a/Makefile b/Makefile index 81bbe18ad..07c2c51aa 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,8 @@ BUILD_FLAGS = "-X '$(REPO)/pkg/config.Version=$(VERSION)' -X '$(REPO)/cli/smartc IMAGE_REPO=nspccdev/neo-go +DISABLE_NEOTEST_COVER=1 + # All of the targets are phony here because we don't really use make dependency # tracking for files .PHONY: build $(BINARY) deps image docker/$(BINARY) image-latest image-push image-push-latest clean-cluster \ diff --git a/pkg/neotest/basic.go b/pkg/neotest/basic.go index 6acc8dbc2..1540af862 100644 --- a/pkg/neotest/basic.go +++ b/pkg/neotest/basic.go @@ -32,20 +32,23 @@ type Executor struct { Validator Signer Committee Signer CommitteeHash util.Uint160 - Contracts map[string]*Contract + // collectCoverage is true if coverage is being collected when running this executor. + collectCoverage bool } // NewExecutor creates a new executor instance from the provided blockchain and committee. +// By default coverage collection is enabled, but only when `go test` is running with coverage enabled. +// Use DisableCoverage and EnableCoverage to stop coverage collection for this executor when not desired. func NewExecutor(t testing.TB, bc *core.Blockchain, validator, committee Signer) *Executor { checkMultiSigner(t, validator) checkMultiSigner(t, committee) return &Executor{ - Chain: bc, - Validator: validator, - Committee: committee, - CommitteeHash: committee.ScriptHash(), - Contracts: make(map[string]*Contract), + Chain: bc, + Validator: validator, + Committee: committee, + CommitteeHash: committee.ScriptHash(), + collectCoverage: isCoverageEnabled(), } } @@ -145,6 +148,7 @@ func (e *Executor) DeployContract(t testing.TB, c *Contract, data any) util.Uint // data is an optional argument to `_deploy`. // It returns the hash of the deploy transaction. func (e *Executor) DeployContractBy(t testing.TB, signer Signer, c *Contract, data any) util.Uint256 { + e.trackCoverage(t, c) tx := e.NewDeployTxBy(t, signer, c, data) e.AddNewBlock(t, tx) e.CheckHalt(t, tx.Hash()) @@ -164,11 +168,22 @@ func (e *Executor) DeployContractBy(t testing.TB, signer Signer, c *Contract, da // DeployContractCheckFAULT compiles and deploys a contract to the bc using the validator // account. It checks that the deploy transaction FAULTed with the specified error. func (e *Executor) DeployContractCheckFAULT(t testing.TB, c *Contract, data any, errMessage string) { + e.trackCoverage(t, c) tx := e.NewDeployTx(t, c, data) e.AddNewBlock(t, tx) e.CheckFault(t, tx.Hash(), errMessage) } +// trackCoverage switches on coverage tracking for provided script if `go test` is running with coverage enabled. +func (e *Executor) trackCoverage(t testing.TB, c *Contract) { + if e.collectCoverage { + addScriptToCoverage(c) + t.Cleanup(func() { + reportCoverage(t) + }) + } +} + // InvokeScript adds a transaction with the specified script to the chain and // returns its hash. It does no faults check. func (e *Executor) InvokeScript(t testing.TB, script []byte, signers []Signer) util.Uint256 { @@ -401,6 +416,10 @@ func (e *Executor) TestInvoke(tx *transaction.Transaction) (*vm.VM, error) { ttx := *tx ic, _ := e.Chain.GetTestVM(trigger.Application, &ttx, b) + if e.collectCoverage { + ic.VM.SetOnExecHook(coverageHook) + } + defer ic.Finalize() ic.VM.LoadWithFlags(tx.Script, callflag.All) @@ -431,3 +450,13 @@ func (e *Executor) GetTxExecResult(t testing.TB, h util.Uint256) *state.AppExecR require.Equal(t, 1, len(aer)) return &aer[0] } + +// EnableCoverage enables coverage collection for this executor, but only when `go test` is running with coverage enabled. +func (e *Executor) EnableCoverage() { + e.collectCoverage = isCoverageEnabled() +} + +// DisableCoverage disables coverage collection for this executor until enabled explicitly through EnableCoverage. +func (e *Executor) DisableCoverage() { + e.collectCoverage = false +} diff --git a/pkg/neotest/client.go b/pkg/neotest/client.go index bd96bab47..3985d8b8c 100644 --- a/pkg/neotest/client.go +++ b/pkg/neotest/client.go @@ -63,6 +63,10 @@ func (c *ContractInvoker) TestInvokeScript(t testing.TB, script []byte, signers } t.Cleanup(ic.Finalize) + if c.collectCoverage { + ic.VM.SetOnExecHook(coverageHook) + } + ic.VM.LoadWithFlags(tx.Script, callflag.All) err = ic.VM.Run() return ic.VM.Estack(), err @@ -78,6 +82,10 @@ func (c *ContractInvoker) TestInvoke(t testing.TB, method string, args ...any) ( } t.Cleanup(ic.Finalize) + if c.collectCoverage { + ic.VM.SetOnExecHook(coverageHook) + } + ic.VM.LoadWithFlags(tx.Script, callflag.All) err = ic.VM.Run() return ic.VM.Estack(), err diff --git a/pkg/neotest/compile.go b/pkg/neotest/compile.go index 70645c11c..2967b2b1e 100644 --- a/pkg/neotest/compile.go +++ b/pkg/neotest/compile.go @@ -16,9 +16,10 @@ import ( // Contract contains contract info for deployment. type Contract struct { - Hash util.Uint160 - NEF *nef.File - Manifest *manifest.Manifest + Hash util.Uint160 + NEF *nef.File + Manifest *manifest.Manifest + DebugInfo *compiler.DebugInfo } // contracts caches the compiled contracts from FS across multiple tests. @@ -36,9 +37,10 @@ func CompileSource(t testing.TB, sender util.Uint160, src io.Reader, opts *compi require.NoError(t, err) return &Contract{ - Hash: state.CreateContractHash(sender, ne.Checksum, m.Name), - NEF: ne, - Manifest: m, + Hash: state.CreateContractHash(sender, ne.Checksum, m.Name), + NEF: ne, + Manifest: m, + DebugInfo: di, } } @@ -73,9 +75,10 @@ func CompileFile(t testing.TB, sender util.Uint160, srcPath string, configPath s require.NoError(t, err) c := &Contract{ - Hash: state.CreateContractHash(sender, ne.Checksum, m.Name), - NEF: ne, - Manifest: m, + Hash: state.CreateContractHash(sender, ne.Checksum, m.Name), + NEF: ne, + Manifest: m, + DebugInfo: di, } contracts[srcPath] = c return c diff --git a/pkg/neotest/coverage.go b/pkg/neotest/coverage.go new file mode 100644 index 000000000..c832b0a43 --- /dev/null +++ b/pkg/neotest/coverage.go @@ -0,0 +1,211 @@ +package neotest + +import ( + "flag" + "fmt" + "io" + "os" + "strconv" + "sync" + "testing" + + "github.com/nspcc-dev/neo-go/pkg/compiler" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm" + "github.com/nspcc-dev/neo-go/pkg/vm/opcode" +) + +const ( + // goCoverProfileFlag specifies the name of `go test` flag that tells it where to save coverage data. + // Neotest coverage can be enabled only when this flag is set. + goCoverProfileFlag = "test.coverprofile" + // disableNeotestCover is name of the environmental variable used to explicitly disable neotest coverage. + disableNeotestCover = "DISABLE_NEOTEST_COVER" +) + +var ( + // coverageLock protects all vars below from concurrent modification when tests are run in parallel. + coverageLock sync.Mutex + // rawCoverage maps script hash to coverage data, collected during testing. + rawCoverage = make(map[util.Uint160]*scriptRawCoverage) + // flagChecked is true if `go test` coverage flag was checked at any point. + flagChecked bool + // coverageEnabled is true if coverage is being collected on test execution. + coverageEnabled bool + // coverProfile specifies the file all coverage data is written to, unless empty. + coverProfile = "" +) + +type scriptRawCoverage struct { + debugInfo *compiler.DebugInfo + offsetsVisited []int +} + +type coverBlock struct { + // Line number for block start. + startLine uint + // Column number for block start. + startCol uint + // Line number for block end. + endLine uint + // Column number for block end. + endCol uint + // Number of statements included in this block. + stmts uint + // Number of times this block was executed. + counts uint +} + +// documentName makes it clear when a `string` maps path to the script file. +type documentName = string + +func isCoverageEnabled() bool { + coverageLock.Lock() + defer coverageLock.Unlock() + + if flagChecked { + return coverageEnabled + } + defer func() { flagChecked = true }() + + var disabledByEnvironment bool + if v, ok := os.LookupEnv(disableNeotestCover); ok { + disabled, err := strconv.ParseBool(v) + if err != nil { + panic(fmt.Sprintf("coverage: error when parsing environment variable '%s', expected bool, but got '%s'", disableNeotestCover, v)) + } + disabledByEnvironment = disabled + } + + var goToolCoverageEnabled bool + flag.VisitAll(func(f *flag.Flag) { + if f.Name == goCoverProfileFlag && f.Value != nil && f.Value.String() != "" { + goToolCoverageEnabled = true + coverProfile = f.Value.String() + } + }) + + coverageEnabled = !disabledByEnvironment && goToolCoverageEnabled + + if coverageEnabled { + // This is needed so go cover tool doesn't overwrite + // the file with our coverage when all tests are done. + err := flag.Set(goCoverProfileFlag, "") + if err != nil { + panic(err) + } + } + + return coverageEnabled +} + +var coverageHook vm.OnExecHook = func(scriptHash util.Uint160, offset int, opcode opcode.Opcode) { + coverageLock.Lock() + defer coverageLock.Unlock() + if cov, ok := rawCoverage[scriptHash]; ok { + cov.offsetsVisited = append(cov.offsetsVisited, offset) + } +} + +func reportCoverage(t testing.TB) { + coverageLock.Lock() + defer coverageLock.Unlock() + f, err := os.Create(coverProfile) + if err != nil { + t.Fatalf("coverage: can't create file '%s' to write coverage report", coverProfile) + } + defer f.Close() + writeCoverageReport(f) +} + +func writeCoverageReport(w io.Writer) { + fmt.Fprintf(w, "mode: set\n") + cover := processCover() + for name, blocks := range cover { + for _, b := range blocks { + c := 0 + if b.counts > 0 { + c = 1 + } + fmt.Fprintf(w, "%s:%d.%d,%d.%d %d %d\n", name, + b.startLine, b.startCol, + b.endLine, b.endCol, + b.stmts, + c, + ) + } + } +} + +func processCover() map[documentName][]coverBlock { + documents := make(map[documentName]struct{}) + for _, scriptRawCoverage := range rawCoverage { + for _, documentName := range scriptRawCoverage.debugInfo.Documents { + documents[documentName] = struct{}{} + } + } + + cover := make(map[documentName][]coverBlock) + + for documentName := range documents { + mappedBlocks := make(map[int]*coverBlock) + + for _, scriptRawCoverage := range rawCoverage { + di := scriptRawCoverage.debugInfo + documentSeqPoints := documentSeqPoints(di, documentName) + + for _, point := range documentSeqPoints { + b := coverBlock{ + startLine: uint(point.StartLine), + startCol: uint(point.StartCol), + endLine: uint(point.EndLine), + endCol: uint(point.EndCol), + stmts: 1 + uint(point.EndLine) - uint(point.StartLine), + counts: 0, + } + mappedBlocks[point.Opcode] = &b + } + } + + for _, scriptRawCoverage := range rawCoverage { + di := scriptRawCoverage.debugInfo + documentSeqPoints := documentSeqPoints(di, documentName) + + for _, offset := range scriptRawCoverage.offsetsVisited { + for _, point := range documentSeqPoints { + if point.Opcode == offset { + mappedBlocks[point.Opcode].counts++ + } + } + } + } + + var blocks []coverBlock + for _, b := range mappedBlocks { + blocks = append(blocks, *b) + } + cover[documentName] = blocks + } + + return cover +} + +func documentSeqPoints(di *compiler.DebugInfo, doc documentName) []compiler.DebugSeqPoint { + var res []compiler.DebugSeqPoint + for _, methodDebugInfo := range di.Methods { + for _, p := range methodDebugInfo.SeqPoints { + if di.Documents[p.Document] == doc { + res = append(res, p) + } + } + } + return res +} + +func addScriptToCoverage(c *Contract) { + coverageLock.Lock() + defer coverageLock.Unlock() + if _, ok := rawCoverage[c.Hash]; !ok { + rawCoverage[c.Hash] = &scriptRawCoverage{debugInfo: c.DebugInfo} + } +} diff --git a/pkg/neotest/doc.go b/pkg/neotest/doc.go index bd223ae56..1e31c4a31 100644 --- a/pkg/neotest/doc.go +++ b/pkg/neotest/doc.go @@ -21,5 +21,13 @@ them in the same package with the smart contract iself can lead to unxpected results if smart contract has any init() functions. If that's the case they will be compiled into the testing binary even when using package_test and their execution can affect tests. See https://github.com/nspcc-dev/neo-go/issues/3120 for details. + +Test coverage for contracts is automatically enabled when `go test` is running with +coverage enabled. When not desired, it can be disabled for any Executor by using +EnableCoverage and DisableCoverage. Be aware that coverage data collected by `go test` +itself will not be saved because it will be replaced with contracts coverage instead. +In case `go test` coverage is wanted DISABLE_NEOTEST_COVER=1 variable can be set. +Coverage is gathered by capturing VM instructions during test contract execution and +mapping them to the contract source code using the DebugInfo information. */ package neotest