tasks: add tests task

Author: nyonson

tasks/src/main.rs

@@ -2,13 +2,15 @@ mod bench;
 mod docs;
 mod environment;
 mod lint;
+mod test;
 mod toolchain;
 
 use clap::{Parser, Subcommand};
 use std::process;
 use xshell::Shell;
 
 use environment::{change_to_repo_root, configure_log_level};
+use toolchain::Toolchain;
 
 #[derive(Parser)]
 #[command(name = "rbmt")]
@@ -28,6 +30,12 @@ enum Commands {
     Docsrs,
     /// Run benchmark tests for all crates.
     Bench,
+    /// Run tests with specified toolchain.
+    Test {
+        /// Toolchain to use: stable, nightly, or msrv.
+        #[arg(value_enum)]
+        toolchain: Toolchain,
+    },
 }
 
 fn main() {
@@ -61,5 +69,11 @@ fn main() {
                 process::exit(1);
             }
         }
+        Commands::Test { toolchain } => {
+            if let Err(e) = test::run(&sh, toolchain) {
+                eprintln!("Error running tests: {}", e);
+                process::exit(1);
+            }
+        }
     }
 }

tasks/src/test.rs

@@ -0,0 +1,253 @@
+//! Build and test tasks with feature matrix testing.
+
+use crate::environment::{get_crate_dirs, quiet_println};
+use crate::quiet_cmd;
+use crate::toolchain::{check_toolchain, Toolchain};
+use serde::Deserialize;
+use std::path::Path;
+use xshell::Shell;
+
+/// Crate test configuration loaded from contrib/test_config.toml.
+#[derive(Debug, Deserialize, Default)]
+struct TestConfig {
+    /// Examples to run with the format "name:feature1 feature2".
+    ///
+    /// # Examples
+    ///
+    /// `["example1:serde", "example2:serde rand"]`
+    #[serde(default)]
+    examples: Vec<String>,
+
+    /// List of individual features to test with the conventional `std` feature enabled.
+    /// Automatically tests feature combinations, alone with `std`, all pairs, and all together.
+    ///
+    /// # Examples
+    ///
+    /// `["serde", "rand"]` tests `std+serde`, `std+rand`, `std+serde+rand`.
+    #[serde(default)]
+    features_with_std: Vec<String>,
+
+    /// List of individual features to test without the `std` feature.
+    /// Automatically tests features combinations, each feature alone,
+    /// all pairs, and all together.
+    ///
+    /// # Examples
+    ///
+    /// `["serde", "rand"]` tests `serde`, `rand`, `serde+rand`.
+    #[serde(default)]
+    features_without_std: Vec<String>,
+
+    /// Exact feature combinations to test.
+    /// Use for crates that don't follow the conventional `std` feature pattern.
+    /// Each inner vector is a list of features to test together. There is
+    /// no automatic combinations of features tests.
+    ///
+    /// # Examples
+    ///
+    /// `[["serde", "rand"], ["rand"]]` tests exactly those two combinations.
+    #[serde(default)]
+    exact_features: Vec<Vec<String>>,
+
+    /// List of individual features to test with the `no-std` feature enabled.
+    /// Only use if your crate has an explicit `no-std` feature (rust-miniscript pattern).
+    /// Automatically tests each feature alone with `no-std`, all pairs, and all together.
+    ///
+    /// # Examples
+    ///
+    /// `["serde", "rand"]` tests `no-std+serde`, `no-std+serde`, `no-std+serde+rand`.
+    #[serde(default)]
+    features_with_no_std: Vec<String>,
+}
+
+impl TestConfig {
+    /// Load test configuration from a crate directory.
+    fn load(crate_dir: &Path) -> Result<Self, Box<dyn std::error::Error>> {
+        let config_path = crate_dir.join("contrib/test_config.toml");
+
+        if !config_path.exists() {
+            // Return default config if file doesn't exist.
+            return Ok(TestConfig::default());
+        }
+
+        let contents = std::fs::read_to_string(&config_path)?;
+        let config: TestConfig = toml::from_str(&contents)?;
+        Ok(config)
+    }
+}
+
+/// Run build and test for all crates with the specified toolchain.
+pub fn run(sh: &Shell, toolchain: Toolchain) -> Result<(), Box<dyn std::error::Error>> {
+    check_toolchain(sh, toolchain)?;
+
+    let crate_dirs = get_crate_dirs(sh)?;
+    quiet_println(&format!("Testing {} crates", crate_dirs.len()));
+
+    for crate_dir in &crate_dirs {
+        quiet_println(&format!("Testing crate: {}", crate_dir));
+
+        let _dir = sh.push_dir(crate_dir);
+        let config = TestConfig::load(Path::new(crate_dir))?;
+
+        do_test(sh, &config)?;
+        do_feature_matrix(sh, &config)?;
+    }
+
+    Ok(())
+}
+
+/// Run basic build, test, and examples.
+fn do_test(sh: &Shell, config: &TestConfig) -> Result<(), Box<dyn std::error::Error>> {
+    quiet_println("Running basic tests");
+
+    // Basic build and test.
+    quiet_cmd!(sh, "cargo build").run()?;
+    quiet_cmd!(sh, "cargo test").run()?;
+
+    // Run examples.
+    for example in &config.examples {
+        let parts: Vec<&str> = example.split(':').collect();
+        if parts.len() != 2 {
+            return Err(format!(
+                "Invalid example format: {}, expected 'name:features'",
+                example
+            )
+            .into());
+        }
+
+        let name = parts[0];
+        let features = parts[1];
+
+        quiet_println(&format!(
+            "Running example {} with features: {}",
+            name, features
+        ));
+        quiet_cmd!(sh, "cargo run --example {name} --features={features}").run()?;
+    }
+
+    Ok(())
+}
+
+/// Run feature matrix tests.
+fn do_feature_matrix(sh: &Shell, config: &TestConfig) -> Result<(), Box<dyn std::error::Error>> {
+    quiet_println("Running feature matrix tests");
+
+    // Handle exact features (for unusual crates).
+    if !config.exact_features.is_empty() {
+        for features in &config.exact_features {
+            let features_str = features.join(" ");
+            quiet_println(&format!("Testing exact features: {}", features_str));
+            quiet_cmd!(
+                sh,
+                "cargo build --no-default-features --features={features_str}"
+            )
+            .run()?;
+            quiet_cmd!(
+                sh,
+                "cargo test --no-default-features --features={features_str}"
+            )
+            .run()?;
+        }
+        return Ok(());
+    }
+
+    // Handle no-std pattern (rust-miniscript).
+    if !config.features_with_no_std.is_empty() {
+        quiet_println("Testing no-std");
+        quiet_cmd!(sh, "cargo build --no-default-features --features=no-std").run()?;
+        quiet_cmd!(sh, "cargo test --no-default-features --features=no-std").run()?;
+
+        loop_features(sh, "no-std", &config.features_with_no_std)?;
+    } else {
+        quiet_println("Testing no-default-features");
+        quiet_cmd!(sh, "cargo build --no-default-features").run()?;
+        quiet_cmd!(sh, "cargo test --no-default-features").run()?;
+    }
+
+    // Test all features.
+    quiet_println("Testing all-features");
+    quiet_cmd!(sh, "cargo build --all-features").run()?;
+    quiet_cmd!(sh, "cargo test --all-features").run()?;
+
+    // Test features with std.
+    if !config.features_with_std.is_empty() {
+        loop_features(sh, "std", &config.features_with_std)?;
+    }
+
+    // Test features without std.
+    if !config.features_without_std.is_empty() {
+        loop_features(sh, "", &config.features_without_std)?;
+    }
+
+    Ok(())
+}
+
+/// Test each feature individually and all combinations of two features.
+///
+/// This implements three feature matrix testing strategies.
+/// 1. All features together.
+/// 2. Each feature individually (only if more than one feature).
+/// 3. All unique pairs of features.
+///
+/// The pair testing catches feature interaction bugs (where two features work
+/// independently, but conflict when combined) while keeping test time manageable.
+fn loop_features(
+    sh: &Shell,
+    base: &str,
+    features: &[String],
+) -> Result<(), Box<dyn std::error::Error>> {
+    let base_flag = if base.is_empty() {
+        String::new()
+    } else {
+        format!("{} ", base)
+    };
+
+    // Test all features together.
+    let all_features = format!("{}{}", base_flag, features.join(" "));
+    quiet_println(&format!("Testing features: {}", all_features.trim()));
+    quiet_cmd!(
+        sh,
+        "cargo build --no-default-features --features={all_features}"
+    )
+    .run()?;
+    quiet_cmd!(
+        sh,
+        "cargo test --no-default-features --features={all_features}"
+    )
+    .run()?;
+
+    // Test each feature individually and all pairs (only if more than one feature).
+    if features.len() > 1 {
+        for i in 0..features.len() {
+            let feature_combo = format!("{}{}", base_flag, features[i]);
+            quiet_println(&format!("Testing features: {}", feature_combo.trim()));
+            quiet_cmd!(
+                sh,
+                "cargo build --no-default-features --features={feature_combo}"
+            )
+            .run()?;
+            quiet_cmd!(
+                sh,
+                "cargo test --no-default-features --features={feature_combo}"
+            )
+            .run()?;
+
+            // Test all pairs with features[i].
+            for j in (i + 1)..features.len() {
+                let feature_combo = format!("{}{} {}", base_flag, features[i], features[j]);
+                quiet_println(&format!("Testing features: {}", feature_combo.trim()));
+                quiet_cmd!(
+                    sh,
+                    "cargo build --no-default-features --features={feature_combo}"
+                )
+                .run()?;
+                quiet_cmd!(
+                    sh,
+                    "cargo test --no-default-features --features={feature_combo}"
+                )
+                .run()?;
+            }
+        }
+    }
+
+    Ok(())
+}

tasks/src/toolchain.rs

@@ -2,7 +2,7 @@ use std::path::Path;
 use xshell::{cmd, Shell};
 
 /// Toolchain requirement for a task.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
 pub enum Toolchain {
     /// Nightly toolchain.
     Nightly,
@@ -92,7 +92,10 @@ fn get_msrv_from_manifest(manifest_path: &Path) -> Result<String, Box<dyn std::e
 }
 
 /// Extract version number from rustc --version output.
-/// Example: "rustc 1.74.0 (79e9716c9 2023-11-13)" -> Some("1.74.0")
+///
+/// # Examples
+///
+/// `"rustc 1.74.0 (79e9716c9 2023-11-13)"` -> `Some("1.74.0")`
 fn extract_version(rustc_version: &str) -> Option<&str> {
     rustc_version.split_whitespace().find_map(|part| {
         // Split off any suffix like "-nightly" or "-beta".