From e3c643055bc68676cb9a8834538aff398416d6f9 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Mon, 4 Jan 2021 09:43:53 -0800 Subject: [PATCH 1/5] Initial matrix --- plugins/hardhat.plugin.js | 15 ++++-- plugins/resources/matrix.js | 71 ++++++++++++++++++++++++++++ plugins/resources/nomiclabs.ui.js | 5 +- plugins/resources/nomiclabs.utils.js | 25 +++++++++- test/units/hardhat/standard.js | 2 +- 5 files changed, 110 insertions(+), 8 deletions(-) create mode 100644 plugins/resources/matrix.js diff --git a/plugins/hardhat.plugin.js b/plugins/hardhat.plugin.js index 3ad19909..a4a1ebd0 100644 --- a/plugins/hardhat.plugin.js +++ b/plugins/hardhat.plugin.js @@ -84,6 +84,7 @@ task("coverage", "Generates a code coverage report for tests") .addOptionalParam("testfiles", ui.flags.file, "", types.string) .addOptionalParam("solcoverjs", ui.flags.solcoverjs, "", types.string) .addOptionalParam('temp', ui.flags.temp, "", types.string) + .addFlag('testMatrix', ui.flags.testMatrix) .setAction(async function(args, env){ let error; @@ -202,6 +203,9 @@ task("coverage", "Generates a code coverage report for tests") ? nomiclabsUtils.getTestFilePaths(args.testfiles) : []; + // Optionally collect tests-per-line-of-code data + nomiclabsUtils.collectTestMatrixData(config, env, api); + try { failedTests = await env.run(TASK_TEST, {testFiles: testfiles}) } catch (e) { @@ -209,10 +213,13 @@ task("coverage", "Generates a code coverage report for tests") } await api.onTestsComplete(config); - // ======== - // Istanbul - // ======== - await api.report(); + // ================================= + // Output (Istanbul or Test Matrix) + // ================================= + (config.testMatrix) + ? await api.saveTestMatrix() + : await api.report(); + await api.onIstanbulComplete(config); } catch(e) { diff --git a/plugins/resources/matrix.js b/plugins/resources/matrix.js new file mode 100644 index 00000000..c47a0b28 --- /dev/null +++ b/plugins/resources/matrix.js @@ -0,0 +1,71 @@ +const mocha = require("mocha"); +const inherits = require("util").inherits; +const Spec = mocha.reporters.Spec; + + +/** + * This file adapted from mocha's stats-collector + * https://github.com/mochajs/mocha/blob/54475eb4ca35a2c9044a1b8c59a60f09c73e6c01/lib/stats-collector.js#L1-L83 + */ +const Date = global.Date; + +/** + * Provides stats such as test duration, number of tests passed / failed etc., by + * listening for events emitted by `runner`. + */ +function mochaStats(runner) { + const stats = { + suites: 0, + tests: 0, + passes: 0, + pending: 0, + failures: 0 + }; + + if (!runner) throw new Error("Missing runner argument"); + + runner.stats = stats; + + runner.on("pass", () => stats.passes++); + runner.on("fail", () => stats.failures++); + runner.on("pending", () => stats.pending++); + runner.on("test end", () => stats.tests++); + + runner.once("start", () => (stats.start = new Date())); + + runner.once("end", function() { + stats.end = new Date(); + stats.duration = stats.end - stats.start; + }); +} + +/** + * Based on the Mocha 'Spec' reporter. Watches an Ethereum test suite run + * and collects data about which tests hit which lines of code. + * This "test matrix" can be used as an input to + * + * + * @param {Object} runner mocha's runner + * @param {Object} options reporter.options (see README example usage) + */ +function Matrix(runner, options) { + // Spec reporter + Spec.call(this, runner, options); + + // Initialize stats for Mocha 6+ epilogue + if (!runner.stats) { + mochaStats(runner); + this.stats = runner.stats; + } + + runner.on("test", (info) => { + console.log('inside test!!') + }); +} + +/** + * Inherit from `Base.prototype`. + */ +inherits(Matrix, Spec); + +module.exports = Matrix; \ No newline at end of file diff --git a/plugins/resources/nomiclabs.ui.js b/plugins/resources/nomiclabs.ui.js index 770fcbad..70270959 100644 --- a/plugins/resources/nomiclabs.ui.js +++ b/plugins/resources/nomiclabs.ui.js @@ -8,7 +8,9 @@ class PluginUI extends UI { super(log); this.flags = { - file: `Path (or glob) defining a subset of tests to run`, + testfiles: `Path (or glob) defining a subset of tests to run`, + + testMatrix: `Generate a json object which maps which unit tests hit which lines of code.`, solcoverjs: `Relative path from working directory to config. ` + `Useful for monorepo packages that share settings.`, @@ -16,7 +18,6 @@ class PluginUI extends UI { temp: `Path to a disposable folder to store compilation artifacts in. ` + `Useful when your test setup scripts include hard-coded paths to ` + `a build directory.`, - } } diff --git a/plugins/resources/nomiclabs.utils.js b/plugins/resources/nomiclabs.utils.js index a3271c61..e1559f67 100644 --- a/plugins/resources/nomiclabs.utils.js +++ b/plugins/resources/nomiclabs.utils.js @@ -38,6 +38,13 @@ function normalizeConfig(config, args={}){ config.solcoverjs = args.solcoverjs config.gasReporter = { enabled: false } + if (config.testMatrix){ + config.measureBranchCoverage = false; + config.measureFunctionCoverage = false; + config.measureModifierCoverage = false; + config.measureStatementCoverage = false; + } + try { const hardhatPackage = require('hardhat/package.json'); if (semver.gt(hardhatPackage.version, '2.0.3')){ @@ -164,6 +171,21 @@ function configureHttpProvider(networkConfig, api, ui){ networkConfig.url = `http://${api.host}:${api.port}`; } +/** + * Configures mocha to generate a json object which maps which tests + * hit which lines of code. + */ +function collectTestMatrixData(config, env, api){ + if (config.testMatrix){ + mochaConfig = env.config.mocha || {}; + mochaConfig.reporter = "./resources/matrix.js"; + mochaConfig.reporterOptions = { + collectTestMatrixData: api.collectTestMatrixData.bind(api) + } + env.config.mocha = mochaConfig; + } +} + /** * Sets the default `from` account field in the network that will be used. * This needs to be done after accounts are fetched from the launched client. @@ -214,6 +236,7 @@ module.exports = { setupBuidlerNetwork, setupHardhatNetwork, getTestFilePaths, - setNetworkFrom + setNetworkFrom, + collectTestMatrixData } diff --git a/test/units/hardhat/standard.js b/test/units/hardhat/standard.js index 1395628f..be516ea1 100644 --- a/test/units/hardhat/standard.js +++ b/test/units/hardhat/standard.js @@ -31,7 +31,7 @@ describe('Hardhat Plugin: standard use cases', function() { mock.clean(); }); - it('simple contract', async function(){ + it.only('simple contract', async function(){ mock.install('Simple', 'simple.js', solcoverConfig); mock.hardhatSetupEnv(this); From f8787fb154654151ab78f369ae52b269375dbf87 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Mon, 4 Jan 2021 14:58:23 -0800 Subject: [PATCH 2/5] Add test matrix generator (--matrix) --- README.md | 4 ++ lib/api.js | 55 +++++++++++++++++++ plugins/hardhat.plugin.js | 7 ++- plugins/resources/matrix.js | 4 +- plugins/resources/nomiclabs.utils.js | 14 ++--- plugins/resources/plugin.utils.js | 7 +++ plugins/resources/truffle.utils.js | 17 +++++- plugins/truffle.plugin.js | 10 +++- scripts/run-metacoin.sh | 10 ++++ scripts/run-nomiclabs.sh | 13 +++++ test/integration/projects/matrix/.solcover.js | 16 ++++++ .../projects/matrix/contracts/MatrixA.sol | 17 ++++++ .../projects/matrix/contracts/MatrixB.sol | 17 ++++++ .../matrix/expectedTestMatrixHardhat.json | 46 ++++++++++++++++ .../matrix/expectedTestMatrixTruffle.json | 46 ++++++++++++++++ .../projects/matrix/hardhat.config.js | 9 +++ .../projects/matrix/test/matrix_a.js | 15 +++++ .../projects/matrix/test/matrix_a_b.js | 30 ++++++++++ .../projects/matrix/truffle-config.js | 10 ++++ test/units/hardhat/flags.js | 19 +++++++ test/units/hardhat/standard.js | 2 +- test/units/truffle/flags.js | 18 ++++++ 22 files changed, 367 insertions(+), 19 deletions(-) create mode 100644 test/integration/projects/matrix/.solcover.js create mode 100644 test/integration/projects/matrix/contracts/MatrixA.sol create mode 100644 test/integration/projects/matrix/contracts/MatrixB.sol create mode 100644 test/integration/projects/matrix/expectedTestMatrixHardhat.json create mode 100644 test/integration/projects/matrix/expectedTestMatrixTruffle.json create mode 100644 test/integration/projects/matrix/hardhat.config.js create mode 100644 test/integration/projects/matrix/test/matrix_a.js create mode 100644 test/integration/projects/matrix/test/matrix_a_b.js create mode 100644 test/integration/projects/matrix/truffle-config.js diff --git a/README.md b/README.md index f251d121..433d22ad 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,8 @@ A working example can be found at [openzeppelin-contracts, here.][35] | solcoverjs | `--solcoverjs ./../.solcover.js` | Relative path from working directory to config. Useful for monorepo packages that share settings. (Path must be "./" prefixed) | | network | `--network development` | Use network settings defined in the Truffle or Buidler config | | temp[*][14] | `--temp build` | :warning: **Caution** :warning: Path to a *disposable* folder to store compilation artifacts in. Useful when your test setup scripts include hard-coded paths to a build directory. [More...][14] | +| matrix | `--matrix` | Generate a JSON object that maps which mocha tests hit which lines of code. (Useful +as an input for some fuzzing, mutation testing and fault-localization algorithms.) [More...][38]| [* Advanced use][14] @@ -123,6 +125,7 @@ module.exports = { | measureStatementCoverage | *boolean* | `true` | Computes statement (in addition to line) coverage. [More...][34] | | measureFunctionCoverage | *boolean* | `true` | Computes function coverage. [More...][34] | | measureModifierCoverage | *boolean* | `true` | Computes each modifier invocation as a code branch. [More...][34] | +| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][38] | | istanbulFolder | *String* | `./coverage` | Folder location for Istanbul coverage reports. | | istanbulReporter | *Array* | `['html', 'lcov', 'text', 'json']` | [Istanbul coverage reporters][2] | | mocha | *Object* | `{ }` | [Mocha options][3] to merge into existing mocha config. `grep` and `invert` are useful for skipping certain tests under coverage using tags in the test descriptions.| @@ -232,4 +235,5 @@ $ yarn [35]: https://github.com/OpenZeppelin/openzeppelin-contracts/blob/e5fbbda9bac49039847a7ed20c1d966766ecc64a/scripts/coverage.js [36]: https://hardhat.org/ [37]: https://github.com/sc-forks/solidity-coverage/blob/master/HARDHAT_README.md +[38]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/advanced.md#generating-a-test-matrix diff --git a/lib/api.js b/lib/api.js index cbc83c4e..aab4caa5 100644 --- a/lib/api.js +++ b/lib/api.js @@ -20,6 +20,7 @@ class API { constructor(config={}) { this.validator = new ConfigValidator() this.config = config || {}; + this.testMatrix = {}; // Validate this.validator.validate(this.config); @@ -30,6 +31,8 @@ class API { this.testsErrored = false; this.cwd = config.cwd || process.cwd(); + this.matrixOutputPath = config.matrixOutputPath || "testMatrix.json"; + this.matrixReporterPath = config.matrixReporterPath || "solidity-coverage/plugins/resources/matrix.js" this.defaultHook = () => {}; this.onServerReady = config.onServerReady || this.defaultHook; @@ -293,6 +296,52 @@ class API { } }*/ + // ========================== + // Test Matrix Data Collector + // ========================== + /** + * @param {Object} testInfo Mocha object passed to reporter 'test end' event + */ + collectTestMatrixData(testInfo){ + const hashes = Object.keys(this.instrumenter.instrumentationData); + const title = testInfo.title; + const file = path.relative(this.cwd, testInfo.file); + + for (const hash of hashes){ + const { + contractPath, + hits, + type, + id + } = this.instrumenter.instrumentationData[hash]; + + if (type === 'line' && hits > 0){ + if (!this.testMatrix[contractPath]){ + this.testMatrix[contractPath] = {}; + } + if (!this.testMatrix[contractPath][id]){ + this.testMatrix[contractPath][id] = []; + } + + // Search for and exclude duplicate entries + let duplicate = false; + for (const item of this.testMatrix[contractPath][id]){ + if (item.title === title && item.file === file){ + duplicate = true; + break; + } + } + + if (!duplicate) { + this.testMatrix[contractPath][id].push({title, file}); + } + + // Reset line data + this.instrumenter.instrumentationData[hash].hits = 0; + } + } + } + // ======== // File I/O // ======== @@ -302,6 +351,12 @@ class API { fs.writeFileSync(covPath, JSON.stringify(data)); } + saveTestMatrix(){ + const matrixPath = path.join(this.cwd, this.matrixOutputPath); + const mapping = this.makeKeysRelative(this.testMatrix, this.cwd); + fs.writeFileSync(matrixPath, JSON.stringify(mapping, null, ' ')); + } + // ===== // Paths // ===== diff --git a/plugins/hardhat.plugin.js b/plugins/hardhat.plugin.js index a4a1ebd0..cfe1379c 100644 --- a/plugins/hardhat.plugin.js +++ b/plugins/hardhat.plugin.js @@ -5,6 +5,7 @@ const PluginUI = require('./resources/nomiclabs.ui'); const pkg = require('./../package.json'); const path = require('path'); +const {inspect} = require('util') const { task, types } = require("hardhat/config"); const { HardhatPluginError } = require("hardhat/plugins") @@ -84,7 +85,7 @@ task("coverage", "Generates a code coverage report for tests") .addOptionalParam("testfiles", ui.flags.file, "", types.string) .addOptionalParam("solcoverjs", ui.flags.solcoverjs, "", types.string) .addOptionalParam('temp', ui.flags.temp, "", types.string) - .addFlag('testMatrix', ui.flags.testMatrix) + .addFlag('matrix', ui.flags.testMatrix) .setAction(async function(args, env){ let error; @@ -204,7 +205,7 @@ task("coverage", "Generates a code coverage report for tests") : []; // Optionally collect tests-per-line-of-code data - nomiclabsUtils.collectTestMatrixData(config, env, api); + nomiclabsUtils.collectTestMatrixData(args, env, api); try { failedTests = await env.run(TASK_TEST, {testFiles: testfiles}) @@ -216,7 +217,7 @@ task("coverage", "Generates a code coverage report for tests") // ================================= // Output (Istanbul or Test Matrix) // ================================= - (config.testMatrix) + (args.matrix) ? await api.saveTestMatrix() : await api.report(); diff --git a/plugins/resources/matrix.js b/plugins/resources/matrix.js index c47a0b28..8701e53c 100644 --- a/plugins/resources/matrix.js +++ b/plugins/resources/matrix.js @@ -58,8 +58,8 @@ function Matrix(runner, options) { this.stats = runner.stats; } - runner.on("test", (info) => { - console.log('inside test!!') + runner.on("test end", (info) => { + options.reporterOptions.collectTestMatrixData(info); }); } diff --git a/plugins/resources/nomiclabs.utils.js b/plugins/resources/nomiclabs.utils.js index e1559f67..f7a1958f 100644 --- a/plugins/resources/nomiclabs.utils.js +++ b/plugins/resources/nomiclabs.utils.js @@ -37,13 +37,7 @@ function normalizeConfig(config, args={}){ config.logger = config.logger ? config.logger : {log: null}; config.solcoverjs = args.solcoverjs config.gasReporter = { enabled: false } - - if (config.testMatrix){ - config.measureBranchCoverage = false; - config.measureFunctionCoverage = false; - config.measureModifierCoverage = false; - config.measureStatementCoverage = false; - } + config.matrix = args.matrix; try { const hardhatPackage = require('hardhat/package.json'); @@ -175,10 +169,10 @@ function configureHttpProvider(networkConfig, api, ui){ * Configures mocha to generate a json object which maps which tests * hit which lines of code. */ -function collectTestMatrixData(config, env, api){ - if (config.testMatrix){ +function collectTestMatrixData(args, env, api){ + if (args.matrix){ mochaConfig = env.config.mocha || {}; - mochaConfig.reporter = "./resources/matrix.js"; + mochaConfig.reporter = api.matrixReporterPath; mochaConfig.reporterOptions = { collectTestMatrixData: api.collectTestMatrixData.bind(api) } diff --git a/plugins/resources/plugin.utils.js b/plugins/resources/plugin.utils.js index 39c5586c..e5f87364 100644 --- a/plugins/resources/plugin.utils.js +++ b/plugins/resources/plugin.utils.js @@ -222,6 +222,13 @@ function loadSolcoverJS(config={}){ coverageConfig.cwd = config.workingDir; coverageConfig.originalContractsDir = config.contractsDir; + if (config.matrix){ + coverageConfig.measureBranchCoverage = false; + coverageConfig.measureFunctionCoverage = false; + coverageConfig.measureModifierCoverage = false; + coverageConfig.measureStatementCoverage = false; + } + // Solidity-Coverage writes to Truffle config config.mocha = config.mocha || {}; diff --git a/plugins/resources/truffle.utils.js b/plugins/resources/truffle.utils.js index 07face1e..36d8b0bf 100644 --- a/plugins/resources/truffle.utils.js +++ b/plugins/resources/truffle.utils.js @@ -198,6 +198,20 @@ function normalizeConfig(config){ return config; } +/** + * Configures mocha to generate a json object which maps which tests + * hit which lines of code. + */ +function collectTestMatrixData(config, api){ + if (config.matrix){ + config.mocha = config.mocha || {}; + config.mocha.reporter = api.matrixReporterPath; + config.mocha.reporterOptions = { + collectTestMatrixData: api.collectTestMatrixData.bind(api) + } + } +} + /** * Replacement logger which filters out compilation warnings triggered by injected trace * function definitions. @@ -222,5 +236,6 @@ module.exports = { setNetworkFrom, loadLibrary, normalizeConfig, - filteredLogger + filteredLogger, + collectTestMatrixData } diff --git a/plugins/truffle.plugin.js b/plugins/truffle.plugin.js index cb6aae22..d526062d 100644 --- a/plugins/truffle.plugin.js +++ b/plugins/truffle.plugin.js @@ -107,6 +107,7 @@ async function plugin(config){ await api.onCompileComplete(config); config.test_files = await truffleUtils.getTestFilePaths(config); + truffleUtils.collectTestMatrixData(config, api); // Run tests try { failures = await truffle.test.run(config) @@ -115,8 +116,13 @@ async function plugin(config){ } await api.onTestsComplete(config); - // Run Istanbul - await api.report(); + // ================================= + // Output (Istanbul or Test Matrix) + // ================================= + (config.matrix) + ? await api.saveTestMatrix() + : await api.report(); + await api.onIstanbulComplete(config); } catch(e){ diff --git a/scripts/run-metacoin.sh b/scripts/run-metacoin.sh index fdc96da3..8bf45ada 100755 --- a/scripts/run-metacoin.sh +++ b/scripts/run-metacoin.sh @@ -53,3 +53,13 @@ if [ ! -d "coverage" ]; then exit 1 fi +npx truffle run coverage --matrix + +# Test that coverage/ was generated +if [ ! -f "testMatrix.json" ]; then + echo "ERROR: no matrix file was created." + exit 1 +fi + +cat testMatrix.json + diff --git a/scripts/run-nomiclabs.sh b/scripts/run-nomiclabs.sh index 72a42efd..c4d826bc 100755 --- a/scripts/run-nomiclabs.sh +++ b/scripts/run-nomiclabs.sh @@ -13,6 +13,13 @@ function verifyCoverageExists { fi } +function verifyMatrixExists { + if [ ! -f "testMatrix.json" ]; then + echo "ERROR: no matrix file was created." + exit 1 + fi +} + # Get rid of any caches sudo rm -rf node_modules echo "NVM CURRENT >>>>>" && nvm current @@ -66,6 +73,12 @@ npx hardhat coverage verifyCoverageExists +npx hardhat coverage --matrix + +verifyMatrixExists + +cat testMatrix.json + echo "" echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" echo "wighawag/hardhat-deploy " diff --git a/test/integration/projects/matrix/.solcover.js b/test/integration/projects/matrix/.solcover.js new file mode 100644 index 00000000..992f82b8 --- /dev/null +++ b/test/integration/projects/matrix/.solcover.js @@ -0,0 +1,16 @@ +// Testing hooks +const fn = (msg, config) => config.logger.log(msg); +const reporterPath = (process.env.TRUFFLE_TEST) + ? "./plugins/resources/matrix.js" + : "../plugins/resources/matrix.js"; + +module.exports = { + // This is loaded directly from `./plugins` during unit tests. The default val is + // "solidity-coverage/plugins/resources/matrix.js" + matrixReporterPath: reporterPath, + matrixOutputPath: "alternateTestMatrix.json", + + skipFiles: ['Migrations.sol'], + silent: process.env.SILENT ? true : false, + istanbulReporter: ['json-summary', 'text'], +} diff --git a/test/integration/projects/matrix/contracts/MatrixA.sol b/test/integration/projects/matrix/contracts/MatrixA.sol new file mode 100644 index 00000000..aeac9bc6 --- /dev/null +++ b/test/integration/projects/matrix/contracts/MatrixA.sol @@ -0,0 +1,17 @@ +pragma solidity ^0.7.0; + + +contract MatrixA { + uint x; + constructor() public { + } + + function sendFn() public { + x = 5; + } + + function callFn() public pure returns (uint){ + uint y = 5; + return y; + } +} diff --git a/test/integration/projects/matrix/contracts/MatrixB.sol b/test/integration/projects/matrix/contracts/MatrixB.sol new file mode 100644 index 00000000..b1981c23 --- /dev/null +++ b/test/integration/projects/matrix/contracts/MatrixB.sol @@ -0,0 +1,17 @@ +pragma solidity ^0.7.0; + + +contract MatrixB { + uint x; + constructor() public { + } + + function sendFn() public { + x = 5; + } + + function callFn() public pure returns (uint){ + uint y = 5; + return y; + } +} diff --git a/test/integration/projects/matrix/expectedTestMatrixHardhat.json b/test/integration/projects/matrix/expectedTestMatrixHardhat.json new file mode 100644 index 00000000..5afc2605 --- /dev/null +++ b/test/integration/projects/matrix/expectedTestMatrixHardhat.json @@ -0,0 +1,46 @@ +{ + "contracts/MatrixA.sol": { + "10": [ + { + "title": "sends to A", + "file": "test/matrix_a_b.js" + }, + { + "title": "sends", + "file": "test/matrix_a.js" + } + ], + "14": [ + { + "title": "calls", + "file": "test/matrix_a.js" + } + ], + "15": [ + { + "title": "calls", + "file": "test/matrix_a.js" + } + ] + }, + "contracts/MatrixB.sol": { + "10": [ + { + "title": "sends to B", + "file": "test/matrix_a_b.js" + } + ], + "14": [ + { + "title": "calls B", + "file": "test/matrix_a_b.js" + } + ], + "15": [ + { + "title": "calls B", + "file": "test/matrix_a_b.js" + } + ] + } +} \ No newline at end of file diff --git a/test/integration/projects/matrix/expectedTestMatrixTruffle.json b/test/integration/projects/matrix/expectedTestMatrixTruffle.json new file mode 100644 index 00000000..f57487ab --- /dev/null +++ b/test/integration/projects/matrix/expectedTestMatrixTruffle.json @@ -0,0 +1,46 @@ +{ + "contracts/MatrixA.sol": { + "10": [ + { + "title": "sends", + "file": "test/matrix_a.js" + }, + { + "title": "sends to A", + "file": "test/matrix_a_b.js" + } + ], + "14": [ + { + "title": "calls", + "file": "test/matrix_a.js" + } + ], + "15": [ + { + "title": "calls", + "file": "test/matrix_a.js" + } + ] + }, + "contracts/MatrixB.sol": { + "10": [ + { + "title": "sends to B", + "file": "test/matrix_a_b.js" + } + ], + "14": [ + { + "title": "calls B", + "file": "test/matrix_a_b.js" + } + ], + "15": [ + { + "title": "calls B", + "file": "test/matrix_a_b.js" + } + ] + } +} \ No newline at end of file diff --git a/test/integration/projects/matrix/hardhat.config.js b/test/integration/projects/matrix/hardhat.config.js new file mode 100644 index 00000000..b402a977 --- /dev/null +++ b/test/integration/projects/matrix/hardhat.config.js @@ -0,0 +1,9 @@ +require("@nomiclabs/hardhat-truffle5"); +require(__dirname + "/../plugins/nomiclabs.plugin"); + +module.exports={ + solidity: { + version: "0.7.3" + }, + logger: process.env.SILENT ? { log: () => {} } : console, +}; diff --git a/test/integration/projects/matrix/test/matrix_a.js b/test/integration/projects/matrix/test/matrix_a.js new file mode 100644 index 00000000..d1cde115 --- /dev/null +++ b/test/integration/projects/matrix/test/matrix_a.js @@ -0,0 +1,15 @@ +const MatrixA = artifacts.require("MatrixA"); + +contract("MatrixA", function(accounts) { + let instance; + + before(async () => instance = await MatrixA.new()) + + it('sends', async function(){ + await instance.sendFn(); + }); + + it('calls', async function(){ + await instance.callFn(); + }) +}); diff --git a/test/integration/projects/matrix/test/matrix_a_b.js b/test/integration/projects/matrix/test/matrix_a_b.js new file mode 100644 index 00000000..6e37de98 --- /dev/null +++ b/test/integration/projects/matrix/test/matrix_a_b.js @@ -0,0 +1,30 @@ +const MatrixA = artifacts.require("MatrixA"); +const MatrixB = artifacts.require("MatrixB"); + +contract("Matrix A and B", function(accounts) { + let instanceA; + let instanceB; + + before(async () => { + instanceA = await MatrixA.new(); + instanceB = await MatrixB.new(); + }) + + it('sends to A', async function(){ + await instanceA.sendFn(); + }); + + // Duplicate test title and file should *not* be duplicated in the output + it('sends to A', async function(){ + await instanceA.sendFn(); + }) + + it('calls B', async function(){ + await instanceB.callFn(); + }) + + it('sends to B', async function(){ + await instanceB.sendFn(); + }); + +}); diff --git a/test/integration/projects/matrix/truffle-config.js b/test/integration/projects/matrix/truffle-config.js new file mode 100644 index 00000000..bac9791d --- /dev/null +++ b/test/integration/projects/matrix/truffle-config.js @@ -0,0 +1,10 @@ +module.exports = { + networks: {}, + mocha: {}, + compilers: { + solc: { + version: "0.7.3" + } + }, + logger: process.env.SILENT ? { log: () => {} } : console, +} diff --git a/test/units/hardhat/flags.js b/test/units/hardhat/flags.js index 5af59490..6127e1ae 100644 --- a/test/units/hardhat/flags.js +++ b/test/units/hardhat/flags.js @@ -172,5 +172,24 @@ describe('Hardhat Plugin: command line options', function() { verify.lineCoverage(expected); shell.rm('.solcover.js'); }); + + it('--matrix', async function(){ + const taskArgs = { + matrix: true + } + + mock.installFullProject('matrix'); + mock.hardhatSetupEnv(this); + + await this.env.run("coverage", taskArgs); + + // Integration test checks output path configurabililty + const altPath = path.join(process.cwd(), './alternateTestMatrix.json'); + const expPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json'); + const producedMatrix = require(altPath) + const expectedMatrix = require(expPath); + + assert.deepEqual(producedMatrix, expectedMatrix); + }); }); diff --git a/test/units/hardhat/standard.js b/test/units/hardhat/standard.js index be516ea1..1395628f 100644 --- a/test/units/hardhat/standard.js +++ b/test/units/hardhat/standard.js @@ -31,7 +31,7 @@ describe('Hardhat Plugin: standard use cases', function() { mock.clean(); }); - it.only('simple contract', async function(){ + it('simple contract', async function(){ mock.install('Simple', 'simple.js', solcoverConfig); mock.hardhatSetupEnv(this); diff --git a/test/units/truffle/flags.js b/test/units/truffle/flags.js index 4506c408..5c233dc4 100644 --- a/test/units/truffle/flags.js +++ b/test/units/truffle/flags.js @@ -271,5 +271,23 @@ describe('Truffle Plugin: command line options', function() { `Should have used default coverage port 8545: ${mock.loggerOutput.val}` ); }); + + it('--matrix', async function(){ + process.env.TRUFFLE_TEST = true; // Path to reporter differs btw HH and Truffle + truffleConfig.matrix = true; + + mock.installFullProject('matrix'); + await plugin(truffleConfig); + + // Integration test checks output path configurabililty + const altPath = path.join(process.cwd(), mock.pathToTemp('./alternateTestMatrix.json')); + const expPath = path.join(process.cwd(), mock.pathToTemp('./expectedTestMatrixHardhat.json')); + + const producedMatrix = require(altPath) + const expectedMatrix = require(expPath); + + assert.deepEqual(producedMatrix, expectedMatrix); + process.env.TRUFFLE_TEST = false; + }); }); From 8c4832bb8d5dec02d45382d19c14a4860cd1895b Mon Sep 17 00:00:00 2001 From: cgewecke Date: Mon, 4 Jan 2021 15:48:49 -0800 Subject: [PATCH 3/5] Add matrix documentation --- docs/advanced.md | 15 ++ docs/matrix.md | 421 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 436 insertions(+) create mode 100644 docs/matrix.md diff --git a/docs/advanced.md b/docs/advanced.md index c267e7f4..8af2aa79 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -97,3 +97,18 @@ Setting the `measureStatementCoverage` and/or `measureFunctionCoverage` options improve performance, lower the cost of execution and minimize complications that arise from `solc`'s limits on how large the compilation payload can be. +## Generating a test matrix + +Some advanced testing strategies benefit from knowing which tests in a suite hit a +specific line of code. Examples include: ++ [mutation testing][22], where this data lets you select the correct subset of tests to check +a mutation with. ++ [fault localization techniques][23], where the complete data set is a key input to algorithms that try +to guess where bugs might exist in a given codebase. + +Running the coverage command with `--matrix` will write [a JSON test matrix][25] which maps greppable +test names to each line of code to a file named `testMatrix.json` in your project's root. + +[22]: https://github.com/JoranHonig/vertigo#vertigo +[23]: http://spideruci.org/papers/jones05.pdf +[25]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/matrix.md diff --git a/docs/matrix.md b/docs/matrix.md new file mode 100644 index 00000000..585186c1 --- /dev/null +++ b/docs/matrix.md @@ -0,0 +1,421 @@ +### Test Matrix Example + +An example of output written to the file `./testMatrix.json` when coverage +is run with the `--matrix` cli flag. (Source project: [sc-forks/hardhat-e2e][1]) + +[1]: https://github.com/sc-forks/hardhat-e2e + + +```js +// Paths are relative to the project root directory +{ + // Solidity file name + "contracts/EtherRouter/EtherRouter.sol": { + + // Line number + "23": [ + { + // Grep-able mocha test title + "title": "Resolves methods routed through an EtherRouter proxy", + + // Selectable mocha test file + "file": "test/etherrouter.js" + } + ], + "42": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ], + "45": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ], + "61": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ] + }, + "contracts/EtherRouter/Factory.sol": { + "19": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ] + }, + "contracts/EtherRouter/Resolver.sol": { + "22": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ], + "26": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ], + "30": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ] + }, + "contracts/MetaCoin.sol": { + "16": [ + { + "title": "should put 10000 MetaCoin in the first account", + "file": "test/metacoin.js" + }, + { + "title": "should call a function that depends on a linked library", + "file": "test/metacoin.js" + }, + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + }, + { + "title": "a and b", + "file": "test/multicontract.js" + } + ], + "20": [ + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + } + ], + "21": [ + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + } + ], + "22": [ + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + } + ], + "23": [ + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + } + ], + "24": [ + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + } + ], + "28": [ + { + "title": "should call a function that depends on a linked library", + "file": "test/metacoin.js" + } + ], + "32": [ + { + "title": "should put 10000 MetaCoin in the first account", + "file": "test/metacoin.js" + }, + { + "title": "should call a function that depends on a linked library", + "file": "test/metacoin.js" + }, + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + } + ] + }, + "contracts/ConvertLib.sol": { + "6": [ + { + "title": "should call a function that depends on a linked library", + "file": "test/metacoin.js" + } + ] + }, + "contracts/MultiContractFile.sol": { + "7": [ + { + "title": "a and b", + "file": "test/multicontract.js" + }, + { + "title": "methods that call methods in other contracts", + "file": "test/variablecosts.js" + } + ], + "15": [ + { + "title": "a and b", + "file": "test/multicontract.js" + } + ] + }, + "contracts/VariableConstructor.sol": { + "8": [ + { + "title": "should should initialize with a short string", + "file": "test/variableconstructor.js" + }, + { + "title": "should should initialize with a medium length string", + "file": "test/variableconstructor.js" + }, + { + "title": "should should initialize with a long string", + "file": "test/variableconstructor.js" + }, + { + "title": "should should initialize with a random length string", + "file": "test/variableconstructor.js" + } + ] + }, + "contracts/VariableCosts.sol": { + "13": [ + { + "title": "should should initialize with a short string", + "file": "test/variableconstructor.js" + }, + { + "title": "should should initialize with a medium length string", + "file": "test/variableconstructor.js" + }, + { + "title": "should should initialize with a long string", + "file": "test/variableconstructor.js" + }, + { + "title": "should should initialize with a random length string", + "file": "test/variableconstructor.js" + }, + { + "title": "should add one", + "file": "test/variablecosts.js" + }, + { + "title": "should add three", + "file": "test/variablecosts.js" + }, + { + "title": "should add even 5!", + "file": "test/variablecosts.js" + }, + { + "title": "should delete one", + "file": "test/variablecosts.js" + }, + { + "title": "should delete three", + "file": "test/variablecosts.js" + }, + { + "title": "should delete five", + "file": "test/variablecosts.js" + }, + { + "title": "should add five and delete one", + "file": "test/variablecosts.js" + }, + { + "title": "should set a random length string", + "file": "test/variablecosts.js" + }, + { + "title": "methods that do not throw", + "file": "test/variablecosts.js" + }, + { + "title": "methods that throw", + "file": "test/variablecosts.js" + }, + { + "title": "methods that call methods in other contracts", + "file": "test/variablecosts.js" + }, + { + "title": "should allow contracts to have identically named methods", + "file": "test/variablecosts.js" + } + ], + "29": [ + { + "title": "should add one", + "file": "test/variablecosts.js" + }, + { + "title": "should add three", + "file": "test/variablecosts.js" + }, + { + "title": "should add even 5!", + "file": "test/variablecosts.js" + }, + { + "title": "should add five and delete one", + "file": "test/variablecosts.js" + } + ], + "30": [ + { + "title": "should add one", + "file": "test/variablecosts.js" + }, + { + "title": "should add three", + "file": "test/variablecosts.js" + }, + { + "title": "should add even 5!", + "file": "test/variablecosts.js" + }, + { + "title": "should add five and delete one", + "file": "test/variablecosts.js" + } + ], + "34": [ + { + "title": "should delete one", + "file": "test/variablecosts.js" + }, + { + "title": "should delete three", + "file": "test/variablecosts.js" + }, + { + "title": "should delete five", + "file": "test/variablecosts.js" + }, + { + "title": "should add five and delete one", + "file": "test/variablecosts.js" + } + ], + "35": [ + { + "title": "should delete one", + "file": "test/variablecosts.js" + }, + { + "title": "should delete three", + "file": "test/variablecosts.js" + }, + { + "title": "should delete five", + "file": "test/variablecosts.js" + }, + { + "title": "should add five and delete one", + "file": "test/variablecosts.js" + } + ], + "43": [ + { + "title": "should set a random length string", + "file": "test/variablecosts.js" + } + ], + "47": [ + { + "title": "methods that do not throw", + "file": "test/variablecosts.js" + }, + { + "title": "methods that throw", + "file": "test/variablecosts.js" + } + ], + "48": [ + { + "title": "methods that do not throw", + "file": "test/variablecosts.js" + } + ], + "52": [ + { + "title": "methods that call methods in other contracts", + "file": "test/variablecosts.js" + } + ], + "53": [ + { + "title": "methods that call methods in other contracts", + "file": "test/variablecosts.js" + } + ], + "54": [ + { + "title": "methods that call methods in other contracts", + "file": "test/variablecosts.js" + } + ] + }, + "contracts/Wallets/Wallet.sol": { + "8": [ + { + "title": "should allow contracts to have identically named methods", + "file": "test/variablecosts.js" + }, + { + "title": "should should allow transfers and sends", + "file": "test/wallet.js" + } + ], + "12": [ + { + "title": "should allow contracts to have identically named methods", + "file": "test/variablecosts.js" + }, + { + "title": "should should allow transfers and sends", + "file": "test/wallet.js" + } + ], + "17": [ + { + "title": "should allow contracts to have identically named methods", + "file": "test/variablecosts.js" + }, + { + "title": "should should allow transfers and sends", + "file": "test/wallet.js" + } + ], + "22": [ + { + "title": "should allow contracts to have identically named methods", + "file": "test/variablecosts.js" + }, + { + "title": "should should allow transfers and sends", + "file": "test/wallet.js" + } + ], + "23": [ + { + "title": "should allow contracts to have identically named methods", + "file": "test/variablecosts.js" + }, + { + "title": "should should allow transfers and sends", + "file": "test/wallet.js" + } + ] + } +} +``` \ No newline at end of file From b040836bb3b16a51565c4187350e5e402d728600 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Mon, 4 Jan 2021 15:53:16 -0800 Subject: [PATCH 4/5] Drop mocha reporter from coverage measurement --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d2fc8088..7ba1fe73 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "scripts": { "nyc": "SILENT=true nyc --exclude '**/sc_temp/**' --exclude '**/test/**'", "test": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit", - "test:ci": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit", + "test:ci": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' --exclude 'plugins/resources/matrix.js' -- mocha test/units/* --timeout 100000 --no-warnings --exit", "test:debug": "node --max-old-space-size=4096 ./node_modules/.bin/mocha test/units/* --timeout 100000 --no-warnings --exit", "netlify": "./scripts/run-netlify.sh" }, From 3dd836112191deda21db3737eab4903f7e88b8d8 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Mon, 4 Jan 2021 16:33:56 -0800 Subject: [PATCH 5/5] Add mocha@7.1.2 to deps (for reporter) --- package.json | 2 +- yarn.lock | 93 +++++++++++++--------------------------------------- 2 files changed, 24 insertions(+), 71 deletions(-) diff --git a/package.json b/package.json index 7ba1fe73..2b2b0df7 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "globby": "^10.0.1", "jsonschema": "^1.2.4", "lodash": "^4.17.15", + "mocha": "7.1.2", "node-emoji": "^1.10.0", "pify": "^4.0.1", "recursive-readdir": "^2.2.2", @@ -55,7 +56,6 @@ "decache": "^4.5.1", "hardhat": "2.0.2", "hardhat-gas-reporter": "^1.0.1", - "mocha": "5.2.0", "nyc": "^14.1.1", "solc": "^0.7.5", "truffle": "5.1.43", diff --git a/yarn.lock b/yarn.lock index aec7bdc8..f0207cef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1502,10 +1502,6 @@ command-exists@^1.2.8: version "1.2.8" resolved "https://registry.yarnpkg.com/command-exists/-/command-exists-1.2.8.tgz#715acefdd1223b9c9b37110a149c6392c2852291" -commander@2.15.1: - version "2.15.1" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.15.1.tgz#df46e867d0fc2aec66a34662b406a9ccafff5b0f" - commander@3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/commander/-/commander-3.0.2.tgz#6837c3fb677ad9933d1cfba42dd14d5117d6b39e" @@ -1723,12 +1719,6 @@ debug@2.6.9, debug@^2.2.0, debug@^2.6.0: dependencies: ms "2.0.0" -debug@3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" - dependencies: - ms "2.0.0" - debug@3.2.6: version "3.2.6" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" @@ -3059,17 +3049,6 @@ glob-parent@~5.1.0: dependencies: is-glob "^4.0.1" -glob@7.1.2: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - glob@7.1.3: version "7.1.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.3.tgz#3960832d3f1574108342dafd3a67b332c0969df1" @@ -3361,10 +3340,6 @@ hasha@^3.0.0: dependencies: is-stream "^1.0.1" -he@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd" - he@1.2.0, he@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" @@ -4337,7 +4312,7 @@ mkdirp-promise@^5.0.1: dependencies: mkdirp "*" -mkdirp@*, mkdirp@0.5.1, mkdirp@0.5.x, mkdirp@^0.5.0: +mkdirp@*, mkdirp@0.5.x, mkdirp@^0.5.0: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" dependencies: @@ -4349,21 +4324,34 @@ mkdirp@0.5.5: dependencies: minimist "^1.2.5" -mocha@5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-5.2.0.tgz#6d8ae508f59167f940f2b5b3c4a612ae50c90ae6" +mocha@7.1.2, mocha@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-7.1.2.tgz#8e40d198acf91a52ace122cd7599c9ab857b29e6" dependencies: + ansi-colors "3.2.3" browser-stdout "1.3.1" - commander "2.15.1" - debug "3.1.0" + chokidar "3.3.0" + debug "3.2.6" diff "3.5.0" escape-string-regexp "1.0.5" - glob "7.1.2" + find-up "3.0.0" + glob "7.1.3" growl "1.10.5" - he "1.1.1" + he "1.2.0" + js-yaml "3.13.1" + log-symbols "3.0.0" minimatch "3.0.4" - mkdirp "0.5.1" - supports-color "5.4.0" + mkdirp "0.5.5" + ms "2.1.1" + node-environment-flags "1.0.6" + object.assign "4.1.0" + strip-json-comments "2.0.1" + supports-color "6.0.0" + which "1.3.1" + wide-align "1.1.3" + yargs "13.3.2" + yargs-parser "13.1.2" + yargs-unparser "1.6.0" mocha@8.1.2: version "8.1.2" @@ -4395,35 +4383,6 @@ mocha@8.1.2: yargs-parser "13.1.2" yargs-unparser "1.6.1" -mocha@^7.1.1: - version "7.1.2" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-7.1.2.tgz#8e40d198acf91a52ace122cd7599c9ab857b29e6" - dependencies: - ansi-colors "3.2.3" - browser-stdout "1.3.1" - chokidar "3.3.0" - debug "3.2.6" - diff "3.5.0" - escape-string-regexp "1.0.5" - find-up "3.0.0" - glob "7.1.3" - growl "1.10.5" - he "1.2.0" - js-yaml "3.13.1" - log-symbols "3.0.0" - minimatch "3.0.4" - mkdirp "0.5.5" - ms "2.1.1" - node-environment-flags "1.0.6" - object.assign "4.1.0" - strip-json-comments "2.0.1" - supports-color "6.0.0" - which "1.3.1" - wide-align "1.1.3" - yargs "13.3.2" - yargs-parser "13.1.2" - yargs-unparser "1.6.0" - mocha@^7.1.2: version "7.2.0" resolved "https://registry.yarnpkg.com/mocha/-/mocha-7.2.0.tgz#01cc227b00d875ab1eed03a75106689cfed5a604" @@ -5935,12 +5894,6 @@ super-split@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/super-split/-/super-split-1.1.0.tgz#43b3ba719155f4d43891a32729d59b213d9155fc" -supports-color@5.4.0: - version "5.4.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.4.0.tgz#1c6b337402c2137605efe19f10fec390f6faab54" - dependencies: - has-flag "^3.0.0" - supports-color@6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.0.0.tgz#76cfe742cf1f41bb9b1c29ad03068c05b4c0e40a"