Merge pull request #6652 from NomicFoundation/coverage-v2

feat: collect instrumentation metadata and node:test coverage data

Author: galargh

.changeset/beige-paws-explode.md

@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/hardhat-node-test-runner": patch
+---
+
+Implemented coverage data collection from the test node task

.changeset/early-dancers-type.md

@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/hardhat-errors": patch
+---
+
+Added the definitions for Hardhat coverage related errors

.changeset/fair-penguins-collect.md

@@ -0,0 +1,5 @@
+---
+"hardhat": patch
+---
+
+Implemented coverage markdown and lcov reporting

.changeset/funny-singers-collect.md

@@ -0,0 +1,5 @@
+---
+"hardhat": patch
+---
+
+Implemented source instrumentation for the coverage data collection

.changeset/sharp-flowers-invent.md

@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/hardhat-mocha": patch
+---
+
+Implemented coverage data collection from the test mocha task

.changeset/ten-apes-camp.md

@@ -0,0 +1,5 @@
+---
+"hardhat": patch
+---
+
+Implemented coverage plugin to enable coverage data collection from the test tasks

v-next/example-project/test/mocha/mocha-test.ts

@@ -27,3 +27,16 @@ describe("Mocha test with chai-matchers", () => {
     expect("0x0000010AB").to.not.hexEqual("0x0010abc");
   });
 });
+
+describe("Rocket test", () => {
+  it("should launch the Apollo 11 rocket", async () => {
+    const connection = await hre.network.connect();
+
+    const Rocket = await connection.ethers.getContractFactory("Rocket");
+    const rocket = await Rocket.deploy("Apollo 11");
+
+    await rocket.launch();
+
+    expect(await rocket.status()).to.equal("lift-off");
+  });
+});

v-next/hardhat-errors/src/descriptors.ts

@@ -121,6 +121,11 @@ export const ERROR_CATEGORIES: {
         max: 1299,
         websiteSubTitle: "Hardhat test plugin errors",
       },
+      COVERAGE: {
+        min: 1300,
+        max: 1399,
+        websiteSubTitle: "Hardhat coverage errors",
+      },
     },
   },
   IGNITION: {
@@ -478,6 +483,12 @@ Please install a version of the peer dependency that meets the plugin's requirem
         websiteTitle: "Plugin hook factory is not a valid file URL",
         websiteDescription: `The loading of a plugin's hook factory failed as the import path is not a valid file:// URL.`,
       },
+      UNEXPECTED_HOOK_PARAM_MODIFICATION: {
+        number: 301,
+        messageTemplate: `Parameter "{paramName}" of hook "{hookCategoryName}#{hookName}" is not allowed to be modified`,
+        websiteTitle: "Unexpected hook parameter modification",
+        websiteDescription: `The parameter is not allowed to be modified`,
+      },
     },
     TASK_DEFINITIONS: {
       INVALID_FILE_ACTION: {
@@ -1227,6 +1238,20 @@ Please use the fully qualified name of the contract to disambiguate it.`,
         websiteDescription: `Cannot determine a test runner for the test files. This may be because the files are not correctly included in the test paths defined by the test plugins in the Hardhat configuration. If they are correctly included, this likely indicates an issue with a plugin failing to detect the files.`,
       },
     },
+    COVERAGE: {
+      SOURCE_NOT_INSTRUMENTED: {
+        number: 1300,
+        messageTemplate: `The source file "{sourceName}" could not be instrumented for coverage.`,
+        websiteTitle: "Source file not instrumented for coverage",
+        websiteDescription: `The source file could not be instrumented for coverage.`,
+      },
+      IMPORT_PATH_ALREADY_DEFINED: {
+        number: 1301,
+        messageTemplate: `The import path "{importPath}" is already defined in the compilation sources`,
+        websiteTitle: "Import path already defined in compilation sources",
+        websiteDescription: `The import path is already defined in the compilation sources`,
+      },
+    },
   },
   IGNITION: {
     GENERAL: {

v-next/hardhat-mocha/src/task-action.ts

@@ -7,6 +7,11 @@ import { fileURLToPath } from "node:url";
 
 import { HardhatError } from "@nomicfoundation/hardhat-errors";
 import { getAllFilesMatching } from "@nomicfoundation/hardhat-utils/fs";
+import {
+  markTestRunDone,
+  markTestRunStart,
+  markTestWorkerDone,
+} from "hardhat/internal/coverage";
 
 interface TestActionArguments {
   testFiles: string[];
@@ -93,10 +98,17 @@ const testWithHardhat: NewTaskActionFunction<TestActionArguments> = async (
   // which supports both ESM and CJS
   await mocha.loadFilesAsync();
 
+  await markTestRunStart("mocha");
+
   const testFailures = await new Promise<number>((resolve) => {
     mocha.run(resolve);
   });
 
+  // NOTE: We execute mocha tests in the main process.
+  await markTestWorkerDone("mocha");
+  // NOTE: This might print a coverage report.
+  await markTestRunDone("mocha");
+
   if (testFailures > 0) {
     process.exitCode = 1;
   }

v-next/hardhat-node-test-runner/package.json

@@ -12,7 +12,8 @@
   "license": "MIT",
   "type": "module",
   "exports": {
-    ".": "./dist/src/index.js"
+    ".": "./dist/src/index.js",
+    "./coverage": "./dist/src/coverage.js"
   },
   "keywords": [
     "ethereum",

v-next/hardhat-node-test-runner/src/coverage.ts

@@ -0,0 +1,7 @@
+import { after } from "node:test";
+
+import { markTestWorkerDone } from "hardhat/internal/coverage";
+
+after(async () => {
+  await markTestWorkerDone("node");
+});

v-next/hardhat-node-test-runner/src/task-action.ts

@@ -9,6 +9,7 @@ import { URL } from "node:url";
 import { hardhatTestReporter } from "@nomicfoundation/hardhat-node-test-reporter";
 import { getAllFilesMatching } from "@nomicfoundation/hardhat-utils/fs";
 import { createNonClosingWriter } from "@nomicfoundation/hardhat-utils/stream";
+import { markTestRunStart, markTestRunDone } from "hardhat/internal/coverage";
 
 interface TestActionArguments {
   testFiles: string[];
@@ -66,8 +67,25 @@ const testWithHardhat: NewTaskActionFunction<TestActionArguments> = async (
     return 0;
   }
 
+  const imports = [];
+
   const tsx = new URL(import.meta.resolve("tsx/esm"));
-  process.env.NODE_OPTIONS = `--import "${tsx.href}"`;
+  imports.push(tsx.href);
+
+  if (hre.globalOptions.coverage === true) {
+    // NOTE: We set the HARDHAT_COVERAGE environment variable here because, as of now,
+    // the global options are not automatically passed to the child processes.
+    process.env.HARDHAT_COVERAGE = "true";
+
+    const coverage = new URL(
+      import.meta.resolve("@nomicfoundation/hardhat-node-test-runner/coverage"),
+    );
+    imports.push(coverage.href);
+  }
+
+  process.env.NODE_OPTIONS = imports
+    .map((href) => `--import "${href}"`)
+    .join(" ");
 
   async function runTests(): Promise<number> {
     let failures = 0;
@@ -106,8 +124,13 @@ const testWithHardhat: NewTaskActionFunction<TestActionArguments> = async (
     return failures;
   }
 
+  await markTestRunStart("node");
+
   const testFailures = await runTests();
 
+  // NOTE: This might print a coverage report.
+  await markTestRunDone("node");
+
   if (testFailures > 0) {
     process.exitCode = 1;
   }

v-next/hardhat/coverage.sol

@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: MIT
+pragma solidity >=0.4.22 <0.9.0;
+
+library __HardhatCoverage {
+  address constant COVERAGE_ADDRESS =
+    0xc0bEc0BEc0BeC0bEC0beC0bEC0bEC0beC0beC0BE;
+
+  function _sendHitImplementation(uint256 coverageId) private view {
+    address coverageAddress = COVERAGE_ADDRESS;
+    /// @solidity memory-safe-assembly
+    assembly {
+      let ptr := mload(0x40)           // Get free memory pointer
+      mstore(ptr, coverageId)          // Store coverageId at free memory
+      pop(
+        staticcall(
+          gas(),
+          coverageAddress,
+          ptr,
+          32,                          // Size of uint256 is 32 bytes
+          0,
+          0
+        )
+      )
+    }
+  }
+
+  function _castToPure(
+    function(uint256) internal view fnIn
+  ) private pure returns (function(uint256) pure fnOut) {
+    assembly {
+      fnOut := fnIn
+    }
+  }
+
+  function sendHit(uint256 coverageId) internal pure {
+    _castToPure(_sendHitImplementation)(coverageId);
+  }
+}

v-next/hardhat/package.json

@@ -34,7 +34,8 @@
     "./types/user-interruptions": "./dist/src/types/user-interruptions.js",
     "./types/utils": "./dist/src/types/utils.js",
     "./types/solidity": "./dist/src/types/solidity.js",
-    "./console.sol": "./console.sol"
+    "./console.sol": "./console.sol",
+    "./internal/coverage": "./dist/src/internal/builtin-plugins/coverage/exports.js"
   },
   "keywords": [
     "ethereum",