Skip to content

feat(forge): add support for mutation tests #10134

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 60 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
60 commits
Select commit Hold shift + click to select a range
c0ffee9
feat: init
simon-something Feb 27, 2025
c0ffeea
chore: lexing/parsing
simon-something Feb 27, 2025
c0ffeef
chore: visiting contracts
simon-something Feb 28, 2025
c0ffeea
chore: wip
simon-something Mar 1, 2025
c0ffee4
chore: wip
simon-something Mar 1, 2025
c0ffee0
chore: wip
simon-something Mar 1, 2025
c0ffee0
chore: visitor not visiting anything
simon-something Mar 1, 2025
c0ffee1
chore: visitor visiting
simon-something Mar 1, 2025
c0ffee0
chore: quick refactor before its too late
simon-something Mar 3, 2025
c0ffee1
chore: quick refactor
simon-something Mar 3, 2025
c0ffee1
chore: mutation gen part1
simon-something Mar 3, 2025
c0ffeed
feat: mutation collection
simon-something Mar 4, 2025
c0ffeee
feat: visitor refactor
simon-something Mar 4, 2025
c0ffeeb
feat: visitor refactor
simon-something Mar 4, 2025
c0ffeef
feat: temp folder mgmt
simon-something Mar 5, 2025
c0ffeec
feat: temp file creation logic
simon-something Mar 6, 2025
c0ffee6
feat: compiling mutants
simon-something Mar 6, 2025
c0ffee2
chore: wip future multithread
simon-something Mar 9, 2025
c0ffeeb
feat: mutation set building
simon-something Mar 10, 2025
c0ffee1
feat: multithread compile
simon-something Mar 11, 2025
c0ffee8
chore: fmt
simon-something Mar 11, 2025
c0ffeeb
chore: fmt
simon-something Mar 11, 2025
c0ffeef
chore: fmt
simon-something Mar 11, 2025
c0ffeee
chore: fmt
simon-something Mar 11, 2025
c0ffeec
feat: refactor for test runner
simon-something Mar 12, 2025
c0ffee4
chore: test runner wip
simon-something Mar 12, 2025
c0ffee1
feat: working poc
simon-something Mar 12, 2025
c0ffee2
chore: doc
simon-something Mar 12, 2025
c0ffee5
chore: fmt
simon-something Mar 12, 2025
c0ffeec
chore: fmt
simon-something Mar 12, 2025
c0ffeed
chore: fmt
simon-something Mar 12, 2025
c0ffee2
feat: assign mut gen
simon-something Mar 12, 2025
c0ffeef
feat: unary mut
simon-something Mar 13, 2025
c0ffeea
feat: binary mut
simon-something Mar 13, 2025
c0ffee4
feat: members unary mut
simon-something Mar 13, 2025
c0ffee4
feat: members unary mut
simon-something Mar 13, 2025
c0ffee1
feat: rm delegatecall mut
simon-something Mar 13, 2025
c0ffeea
chore: refactor modular mut wip
simon-something Mar 13, 2025
c0ffee0
chore: refactor modular mut wip
simon-something Mar 13, 2025
c0ffee5
chore: refactor modular mut wip
simon-something Mar 14, 2025
c0ffeee
feat: mutator assign trait
simon-something Mar 17, 2025
c0ffee3
chore: fmt
simon-something Mar 17, 2025
c0ffee2
chore: fmt
simon-something Mar 17, 2025
c0ffee6
feat: bin mutator
simon-something Mar 18, 2025
c0ffee1
feat: other mutator mod
simon-something Mar 19, 2025
c0ffeea
feat: visitor refactor registry
simon-something Mar 20, 2025
0060f79
Merge branch 'master' into feat/mutation-tests-wonder
simon-something Mar 20, 2025
c0ffee8
chore: merge fix
simon-something Mar 20, 2025
39e5f62
Merge pull request #2 from defi-wonderland/feat/mutation-tests-wonder
simon-something Mar 20, 2025
1a90f9c
Merge branch 'master' into master
simon-something Mar 20, 2025
18fb058
chore: comment
simon-something Mar 20, 2025
c0ffee1
fix: solar visitor use
simon-something Mar 20, 2025
c0ffee8
chore: clippy and min refactors
simon-something Mar 21, 2025
c0ffeef
feat: assign mutator tests
simon-something Mar 21, 2025
c0ffeea
feat: test gen mut ident
simon-something Mar 21, 2025
5b67d3f
feat: test binop and delete expr
simon-something Mar 25, 2025
7690e44
feat: test delegate unaryop mut
simon-something Mar 25, 2025
8137d21
feat: test unaryop mut
simon-something Mar 25, 2025
15b8cec
feat: wip generic test
simon-something Mar 25, 2025
47aa3ca
Merge remote-tracking branch 'upstream/master'
simon-something Mar 28, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions crates/forge/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,17 @@ indicatif.workspace = true
inferno = { version = "0.12", default-features = false }
itertools.workspace = true
parking_lot.workspace = true
rand.workspace = true
regex = { workspace = true, default-features = false }
reqwest = { workspace = true, features = ["json"] }
revm.workspace = true
semver.workspace = true
serde_json.workspace = true
similar = { version = "2", features = ["inline"] }
solang-parser.workspace = true
solar-parse.workspace = true
strum = { workspace = true, features = ["derive"] }
tempfile.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["time"] }
toml = { workspace = true, features = ["preserve_order"] }
Expand All @@ -84,6 +87,7 @@ watchexec-events = "5.0"
watchexec-signals = "4.0"
clearscreen = "4.0"
evm-disassembler.workspace = true
num-bigint = "0.4"

# doc server
axum = { workspace = true, features = ["ws"] }
Expand Down
101 changes: 97 additions & 4 deletions crates/forge/src/cmd/test/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use crate::{
decode::decode_console_logs,
gas_report::GasReport,
multi_runner::matches_contract,
mutation::MutationHandler,
result::{SuiteResult, TestOutcome, TestStatus},
traces::{
debug::{ContractSources, DebugTraceIdentifier},
Expand All @@ -18,7 +19,7 @@ use clap::{Parser, ValueHint};
use eyre::{bail, Context, OptionExt, Result};
use foundry_cli::{
opts::{BuildOpts, GlobalArgs},
utils::{self, LoadConfig},
utils::{self, FoundryPathExt, LoadConfig},
};
use foundry_common::{compile::ProjectCompiler, evm::EvmArgs, fs, shell, TestFunctionExt};
use foundry_compilers::{
Expand Down Expand Up @@ -193,6 +194,12 @@ pub struct TestArgs {

#[command(flatten)]
pub watch: WatchArgs,

/// Enable mutation testing.
/// If passed without arguments, all contracts will be tested.
/// If passed with file paths, only those files will be tested.
#[arg(long, num_args(0..), value_name = "PATH")]
pub mutate: Option<Vec<PathBuf>>,
}

impl TestArgs {
Expand Down Expand Up @@ -291,6 +298,13 @@ impl TestArgs {
config.invariant.gas_report_samples = 0;
}

let should_mutate = self.mutate.is_some();

// Mutation test uses cache to avoid recompiling non-mutated contracts -> force it
if should_mutate && !config.cache {
config.cache = true;
}

// Install missing dependencies.
if install::install_missing_dependencies(&mut config) && config.auto_detect_remappings {
// need to re-configure here to also catch additional remappings
Expand Down Expand Up @@ -349,10 +363,11 @@ impl TestArgs {
.with_fork(evm_opts.get_fork(&config, env.clone()))
.enable_isolation(evm_opts.isolate)
.odyssey(evm_opts.odyssey)
.build::<MultiCompiler>(project_root, &output, env, evm_opts)?;
.build::<MultiCompiler>(project_root, &output, env.clone(), evm_opts.clone())?;

let libraries = runner.libraries.clone();
let mut outcome = self.run_tests(runner, config, verbosity, &filter, &output).await?;
let mut outcome =
self.run_tests(runner, config.clone(), verbosity, &filter, &output).await?;

if should_draw {
let (suite_name, test_name, mut test_result) =
Expand Down Expand Up @@ -416,13 +431,91 @@ impl TestArgs {
}

let mut debugger = builder.build();
if let Some(dump_path) = self.dump {
if let Some(dump_path) = self.dump.clone() {
debugger.dump_to_file(&dump_path)?;
} else {
debugger.try_run_tui()?;
}
}

// All test have been run once before reaching this point
if should_mutate {
// check outcome here, stop if any test failed
// @todo rather set non-allowed failed tests in config and ensure_ok() here?
// @todo other checks: no fork (or just exclude based on clap arg?)
if outcome.failed() > 0 {
eyre::bail!("Cannot run mutation testing with failed tests");
}

let mutate_paths = if self.mutate.as_ref().unwrap().is_empty() {
// If --mutate is passed without arguments, list all non-test contracts
source_files_iter(&project.paths.sources, MultiCompilerLanguage::FILE_EXTENSIONS)
.filter(|entry| {
entry.is_sol() && !entry.is_sol_test() // @todo filter out interfaces here?
// we do it in lexing for now
})
.collect()
} else {
// If --mutate is passed with arguments, use those paths
self.mutate.as_ref().unwrap().clone()
};

sh_println!("Running mutation tests...").unwrap();
for path in mutate_paths {
let mut handler = MutationHandler::new(path, config.clone());

handler.read_source_contract()?;
handler.generate_ast().await;
handler.create_mutation_folders();

let mutants = handler.generate_and_compile().await;

// @todo multithread here? -> tests are async already
for mutant in mutants {
if let Some(compile_output) = mutant.1 {
let mutant_path = mutant.0.path.clone();

let mut new_config = (*config).clone();

new_config.root = mutant_path.clone();
new_config.src = mutant_path.clone().join("src");
new_config.out = mutant_path.clone().join("out");
new_config.test = mutant_path.clone().join("test");

new_config.cache_path = mutant_path.clone().join("cache");

let new_config = Arc::new(new_config);
let new_filter = self.filter(&new_config).unwrap();

let mut runner = MultiContractRunnerBuilder::new(new_config.clone())
.set_debug(false)
.initial_balance(evm_opts.initial_balance)
.evm_spec(config.evm_spec_id())
.sender(evm_opts.sender)
.odyssey(evm_opts.odyssey)
.build::<MultiCompiler>(
&mutant_path,
&compile_output,
env.clone(),
evm_opts.clone(),
)?;

let results = runner.test_collect(&new_filter);

let outcome = TestOutcome::new(results, self.allow_failure);

if outcome.failed() != 0 {
sh_println!("Mutation: Dead")?;
} else {
sh_println!("Mutation: Survived")?;
}
} else {
sh_println!("Mutation: Invalid")?;
}
}
}
}

Ok(outcome)
}

Expand Down
2 changes: 2 additions & 0 deletions crates/forge/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ pub mod gas_report;
pub mod multi_runner;
pub use multi_runner::{MultiContractRunner, MultiContractRunnerBuilder};

pub mod mutation;

mod runner;
pub use runner::ContractRunner;

Expand Down
Loading