fix(forge): show logs and coverage for table tests (#11575)

Author: grandizzy

crates/evm/fuzz/src/lib.rs

@@ -199,7 +199,7 @@ impl fmt::Display for BaseCounterExample {
 }
 
 /// The outcome of a fuzz test
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct FuzzTestResult {
     /// we keep this for the debugger
     pub first_case: FuzzCase,

crates/forge/src/result.rs

@@ -684,6 +684,33 @@ impl TestResult {
         self.gas_report_traces = gas_report_traces;
     }
 
+    /// Returns the result for a table test. Merges table test execution results (logs, labeled
+    /// addresses, traces and coverages) in initial setup results.
+    pub fn table_result(&mut self, result: FuzzTestResult) {
+        self.kind = TestKind::Table {
+            median_gas: result.median_gas(false),
+            mean_gas: result.mean_gas(false),
+            runs: result.gas_by_case.len(),
+        };
+
+        // Record logs, labels, traces and merge coverages.
+        extend!(self, result, TraceKind::Execution);
+
+        self.status = if result.skipped {
+            TestStatus::Skipped
+        } else if result.success {
+            TestStatus::Success
+        } else {
+            TestStatus::Failure
+        };
+        self.reason = result.reason;
+        self.counterexample = result.counterexample;
+        self.duration = Duration::default();
+        self.gas_report_traces = result.gas_report_traces.into_iter().map(|t| vec![t]).collect();
+        self.breakpoints = result.breakpoints.unwrap_or_default();
+        self.deprecated_cheatcodes = result.deprecated_cheatcodes;
+    }
+
     /// Returns `true` if this is the result of a fuzz test
     pub fn is_fuzz(&self) -> bool {
         matches!(self.kind, TestKind::Fuzz { .. })
@@ -724,6 +751,11 @@ pub enum TestKindReport {
         metrics: Map<String, InvariantMetrics>,
         failed_corpus_replays: usize,
     },
+    Table {
+        runs: usize,
+        mean_gas: u64,
+        median_gas: u64,
+    },
 }
 
 impl fmt::Display for TestKindReport {
@@ -752,6 +784,9 @@ impl fmt::Display for TestKindReport {
                     write!(f, "(runs: {runs}, calls: {calls}, reverts: {reverts})")
                 }
             }
+            Self::Table { runs, mean_gas, median_gas } => {
+                write!(f, "(runs: {runs}, μ: {mean_gas}, ~: {median_gas})")
+            }
         }
     }
 }
@@ -762,7 +797,7 @@ impl TestKindReport {
         match *self {
             Self::Unit { gas } => gas,
             // We use the median for comparisons
-            Self::Fuzz { median_gas, .. } => median_gas,
+            Self::Fuzz { median_gas, .. } | Self::Table { median_gas, .. } => median_gas,
             // We return 0 since it's not applicable
             Self::Invariant { .. } => 0,
         }
@@ -791,6 +826,8 @@ pub enum TestKind {
         metrics: Map<String, InvariantMetrics>,
         failed_corpus_replays: usize,
     },
+    /// A table test.
+    Table { runs: usize, mean_gas: u64, median_gas: u64 },
 }
 
 impl Default for TestKind {
@@ -821,6 +858,9 @@ impl TestKind {
                     failed_corpus_replays: *failed_corpus_replays,
                 }
             }
+            Self::Table { runs, mean_gas, median_gas } => {
+                TestKindReport::Table { runs: *runs, mean_gas: *mean_gas, median_gas: *median_gas }
+            }
         }
     }
 }

crates/forge/src/runner.rs

@@ -2,7 +2,8 @@
 
 use crate::{
     MultiContractRunner, TestFilter,
-    fuzz::BaseCounterExample,
+    coverage::HitMaps,
+    fuzz::{BaseCounterExample, FuzzTestResult},
     multi_runner::{TestContract, TestRunnerConfig},
     progress::{TestsProgress, start_fuzz_progress},
     result::{SuiteResult, TestResult, TestSetup},
@@ -640,6 +641,8 @@ impl<'a> FunctionRunner<'a> {
             fixtures_len as u32,
         );
 
+        let mut result = FuzzTestResult::default();
+
         for i in 0..fixtures_len {
             if self.tcfg.fail_fast.should_stop() {
                 return self.result;
@@ -671,24 +674,33 @@ impl<'a> FunctionRunner<'a> {
                 }
             };
 
+            result.gas_by_case.push((raw_call_result.gas_used, raw_call_result.stipend));
+            result.logs.extend(raw_call_result.logs.clone());
+            result.labels.extend(raw_call_result.labels.clone());
+            HitMaps::merge_opt(&mut result.line_coverage, raw_call_result.line_coverage.clone());
+
             let is_success =
                 self.executor.is_raw_call_mut_success(self.address, &mut raw_call_result, false);
             // Record counterexample if test fails.
             if !is_success {
-                self.result.counterexample =
+                result.counterexample =
                     Some(CounterExample::Single(BaseCounterExample::from_fuzz_call(
                         Bytes::from(func.abi_encode_input(&args).unwrap()),
                         args,
                         raw_call_result.traces.clone(),
                     )));
-                self.result.single_result(false, reason, raw_call_result);
+                result.reason = reason;
+                result.traces = raw_call_result.traces;
+                self.result.table_result(result);
                 return self.result;
             }
 
             // If it's the last iteration and all other runs succeeded, then use last call result
             // for logs and traces.
             if i == fixtures_len - 1 {
-                self.result.single_result(true, None, raw_call_result);
+                result.success = true;
+                result.traces = raw_call_result.traces;
+                self.result.table_result(result);
                 return self.result;
             }
         }

crates/forge/tests/it/table.rs

@@ -69,25 +69,25 @@ Compiler run successful!
 
 Ran 8 tests for test/CounterTable.t.sol:CounterTableTest
 [FAIL: 2 fixtures defined for diffSwap (expected 10)] tableMultipleParamsDifferentFixturesFail(uint256,bool) ([GAS])
-[FAIL: Cannot swap; counterexample: calldata=0x717892ca00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001 args=[1, true]] tableMultipleParamsFail(uint256,bool) ([GAS])
+[FAIL: Cannot swap; counterexample: calldata=0x717892ca00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001 args=[1, true]] tableMultipleParamsFail(uint256,bool) (runs: 1, [AVG_GAS])
 Traces:
   [..] CounterTableTest::tableMultipleParamsFail(1, true)
     └─ ← [Revert] Cannot swap
 
 [FAIL: No fixture defined for param noSwap] tableMultipleParamsNoParamFail(uint256,bool) ([GAS])
-[PASS] tableMultipleParamsPass(uint256,bool) ([GAS])
+[PASS] tableMultipleParamsPass(uint256,bool) (runs: 10, [AVG_GAS])
 Traces:
   [..] CounterTableTest::tableMultipleParamsPass(10, true)
     ├─ [..] Counter::increment()
     │   └─ ← [Stop]
     └─ ← [Stop]
 
-[FAIL: Amount cannot be 10; counterexample: calldata=0x44fa2375000000000000000000000000000000000000000000000000000000000000000a args=[10]] tableSingleParamFail(uint256) ([GAS])
+[FAIL: Amount cannot be 10; counterexample: calldata=0x44fa2375000000000000000000000000000000000000000000000000000000000000000a args=[10]] tableSingleParamFail(uint256) (runs: 10, [AVG_GAS])
 Traces:
   [..] CounterTableTest::tableSingleParamFail(10)
     └─ ← [Revert] Amount cannot be 10
 
-[PASS] tableSingleParamPass(uint256) ([GAS])
+[PASS] tableSingleParamPass(uint256) (runs: 10, [AVG_GAS])
 Traces:
   [..] CounterTableTest::tableSingleParamPass(10)
     ├─ [..] Counter::increment()
@@ -103,13 +103,117 @@ Ran 1 test suite [ELAPSED]: 2 tests passed, 6 failed, 0 skipped (8 total tests)
 Failing tests:
 Encountered 6 failing tests in test/CounterTable.t.sol:CounterTableTest
 [FAIL: 2 fixtures defined for diffSwap (expected 10)] tableMultipleParamsDifferentFixturesFail(uint256,bool) ([GAS])
-[FAIL: Cannot swap; counterexample: calldata=0x717892ca00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001 args=[1, true]] tableMultipleParamsFail(uint256,bool) ([GAS])
+[FAIL: Cannot swap; counterexample: calldata=0x717892ca00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001 args=[1, true]] tableMultipleParamsFail(uint256,bool) (runs: 1, [AVG_GAS])
 [FAIL: No fixture defined for param noSwap] tableMultipleParamsNoParamFail(uint256,bool) ([GAS])
-[FAIL: Amount cannot be 10; counterexample: calldata=0x44fa2375000000000000000000000000000000000000000000000000000000000000000a args=[10]] tableSingleParamFail(uint256) ([GAS])
+[FAIL: Amount cannot be 10; counterexample: calldata=0x44fa2375000000000000000000000000000000000000000000000000000000000000000a args=[10]] tableSingleParamFail(uint256) (runs: 10, [AVG_GAS])
 [FAIL: Table test should have at least one parameter] tableWithNoParamFail() ([GAS])
 [FAIL: Table test should have at least one fixture] tableWithParamNoFixtureFail(uint256) ([GAS])
 
 Encountered a total of 6 failing tests, 2 tests succeeded
 
 "#]]);
 });
+
+// Table tests should show logs and contribute to coverage.
+// <https://github.com/foundry-rs/foundry/issues/11066>
+forgetest_init!(should_show_logs_and_add_coverage, |prj, cmd| {
+    prj.wipe_contracts();
+    prj.add_source(
+        "Counter.sol",
+        r#"
+contract Counter {
+    uint256 public number;
+
+    function setNumber(uint256 a, uint256 b) public {
+        if (a == 1) {
+            number = b + 1;
+        } else if (a == 2) {
+            number = b + 2;
+        } else if (a == 3) {
+            number = b + 3;
+        } else {
+            number = a + b;
+        }
+    }
+}
+    "#,
+    );
+    prj.add_test(
+        "CounterTest.t.sol",
+        r#"
+import "forge-std/Test.sol";
+import {Counter} from "../src/Counter.sol";
+
+contract CounterTest is Test {
+    struct TestCase {
+        uint256 a;
+        uint256 b;
+        uint256 expected;
+    }
+
+    Counter public counter;
+
+    function setUp() public {
+        counter = new Counter();
+    }
+
+    function fixtureNumbers() public pure returns (TestCase[] memory) {
+        TestCase[] memory entries = new TestCase[](4);
+        entries[0] = TestCase(1, 5, 6);
+        entries[1] = TestCase(2, 10, 12);
+        entries[2] = TestCase(3, 11, 14);
+        entries[3] = TestCase(4, 11, 15);
+        return entries;
+    }
+
+    function tableSetNumberTest(TestCase memory numbers) public {
+        console.log("expected", numbers.expected);
+        counter.setNumber(numbers.a, numbers.b);
+        require(counter.number() == numbers.expected, "test failed");
+    }
+}
+    "#,
+    );
+
+    cmd.args(["test", "-vvv"]).assert_success().stdout_eq(str![[r#"
+[COMPILING_FILES] with [SOLC_VERSION]
+[SOLC_VERSION] [ELAPSED]
+Compiler run successful!
+
+Ran 1 test for test/CounterTest.t.sol:CounterTest
+[PASS] tableSetNumberTest((uint256,uint256,uint256)) (runs: 4, [AVG_GAS])
+Logs:
+  expected 6
+  expected 12
+  expected 14
+  expected 15
+
+Suite result: ok. 1 passed; 0 failed; 0 skipped; [ELAPSED]
+
+Ran 1 test suite [ELAPSED]: 1 tests passed, 0 failed, 0 skipped (1 total tests)
+
+"#]]);
+
+    cmd.forge_fuse().args(["coverage"]).assert_success().stdout_eq(str![[r#"
+[COMPILING_FILES] with [SOLC_VERSION]
+[SOLC_VERSION] [ELAPSED]
+Compiler run successful!
+Analysing contracts...
+Running tests...
+
+Ran 1 test for test/CounterTest.t.sol:CounterTest
+[PASS] tableSetNumberTest((uint256,uint256,uint256)) (runs: 4, [AVG_GAS])
+Suite result: ok. 1 passed; 0 failed; 0 skipped; [ELAPSED]
+
+Ran 1 test suite [ELAPSED]: 1 tests passed, 0 failed, 0 skipped (1 total tests)
+
+╭-----------------+---------------+---------------+---------------+---------------╮
+| File            | % Lines       | % Statements  | % Branches    | % Funcs       |
++=================================================================================+
+| src/Counter.sol | 100.00% (8/8) | 100.00% (7/7) | 100.00% (6/6) | 100.00% (1/1) |
+|-----------------+---------------+---------------+---------------+---------------|
+| Total           | 100.00% (8/8) | 100.00% (7/7) | 100.00% (6/6) | 100.00% (1/1) |
+╰-----------------+---------------+---------------+---------------+---------------╯
+
+"#]]);
+});