feat: Turn experimental options into -Z unstable flags and options

This is modeled after rustc, and primarily Cargo.

Author: zalanlevai

Cargo.lock

@@ -1,11 +1,14 @@
 #![feature(if_let_guard)]
+#![feature(trim_prefix_suffix)]
 
 use std::collections::HashSet;
 use std::env;
 use std::fs;
 use std::path::PathBuf;
 use std::process::{self, Command};
 
+use mutest_driver_cli::{UnstableFlag, UnstableOption};
+
 pub mod build {
     pub const RUST_TOOLCHAIN_VERSION: &str = env!("RUST_TOOLCHAIN_VERSION");
 }
@@ -21,13 +24,43 @@ fn strip_arg(args: &mut Vec<String>, has_value: bool, short_arg: Option<&str>, l
             .or_else(|| long_arg.as_deref().and_then(|v| arg.strip_prefix(v)));
 
         match arg_without_prefix.map(|v| has_value && !v.trim_start().starts_with("=") && i + 1 < args.len()) {
-            Some(true) => { args.splice(i..=(i + 1), None); }
+            Some(true) => { args.splice(i..=(i + 1), []); }
             Some(false) => { args.remove(i); }
             None => i += 1,
         }
     }
 }
 
+fn strip_arg_value_occurrences(args: &mut Vec<String>, short_arg: Option<&str>, long_arg: Option<&str>, value: &str) {
+    let short_arg = short_arg.map(|v| format!("-{v}"));
+    let long_arg = long_arg.map(|v| format!("--{v}"));
+
+    let mut i = 0;
+    while i < args.len() {
+        let arg = &args[i];
+
+        match () {
+            _ if let Some(short_arg_without_prefix) = short_arg.as_deref().and_then(|v| arg.strip_prefix(v)) => {
+                let short_arg_inline_value = short_arg_without_prefix.trim_prefix('=');
+                match (short_arg_inline_value.is_empty(), args.get(i + 1)) {
+                    (false, _) if short_arg_inline_value == value => { args.remove(i); }
+                    (true, Some(v)) if v == value => { args.splice(i..=(i + 1), []); }
+                    _ => { i += 1; }
+                }
+            }
+            _ if let Some(long_arg_without_prefix) = long_arg.as_deref().and_then(|v| arg.strip_prefix(v)) => {
+                let long_arg_inline_value = long_arg_without_prefix.strip_prefix('=');
+                match (long_arg_inline_value, args.get(i + 1)) {
+                    (Some(v), _) if v == value => { args.remove(i); }
+                    (None, Some(v)) if v == value => { args.splice(i..=(i + 1), []); }
+                    _ => { i += 1; }
+                }
+            }
+            _ => { i += 1; }
+        }
+    }
+}
+
 #[test]
 fn test_strip_arg() {
     let mut args = vec!["--lib".to_owned()];
@@ -51,6 +84,38 @@ fn test_strip_arg() {
     assert_eq!(&["--metadata-out-root-dir=target/mutest/json".to_owned(), "--print=code".to_owned()] as &[String], &args[..]);
 }
 
+#[test]
+fn test_strip_arg_value_occurences() {
+    let mut args = vec!["-Z".to_owned(), "write-json-eval-stream".to_owned()];
+    strip_arg_value_occurrences(&mut args, Some("Z"), None, "write-json-eval-stream");
+    assert_eq!(&[] as &[String], &args[..]);
+
+    let mut args = vec!["-Zwrite-json-eval-stream".to_owned()];
+    strip_arg_value_occurrences(&mut args, Some("Z"), None, "write-json-eval-stream");
+    assert_eq!(&[] as &[String], &args[..]);
+
+    let mut args = vec!["-Z=write-json-eval-stream".to_owned()];
+    strip_arg_value_occurrences(&mut args, Some("Z"), None, "write-json-eval-stream");
+    assert_eq!(&[] as &[String], &args[..]);
+
+    let mut args = vec!["-Z".to_owned(), "feature-a".to_owned(), "-Z".to_owned(), "feature-b".to_owned(), "-Z".to_owned(), "feature-c".to_owned()];
+    strip_arg_value_occurrences(&mut args, Some("Z"), None, "feature-b");
+    assert_eq!(&["-Z".to_owned(), "feature-a".to_owned(), "-Z".to_owned(), "feature-c".to_owned()] as &[String], &args[..]);
+
+    let mut args = vec!["-Z".to_owned(), "feature-b".to_owned(), "-Z".to_owned(), "feature-a".to_owned(), "-Z".to_owned(), "feature-b".to_owned()];
+    strip_arg_value_occurrences(&mut args, Some("Z"), None, "feature-b");
+    assert_eq!(&["-Z".to_owned(), "feature-a".to_owned()] as &[String], &args[..]);
+}
+
+const RUN_UNSTABLE_FLAGS: &[UnstableFlag] = mutest_driver_cli::extend_const_slice!(mutest_driver_cli::UNSTABLE_FLAGS: &[UnstableFlag], &[
+    // NOTE: Whenever these change, the `strip_arg_value_occurrences` calls in `run_cargo_with_mutest_driver` have to be updated as well.
+    UnstableFlag::new("write-json-eval-stream", Some("During evaluation, write JSONL stream file into JSON output directory specified by `--metadata-out-root-dir`.")),
+]);
+
+const RUN_UNSTABLE_OPTIONS: &[UnstableOption] = mutest_driver_cli::extend_const_slice!(mutest_driver_cli::UNSTABLE_OPTIONS: &[UnstableOption], &[
+    // No Cargo or evaluation-specific unstable options at the moment.
+]);
+
 mod run_isolate {
     mutest_driver_cli::exclusive_opts! { pub(crate) possible_values where
         UNSAFE = "unsafe"; ["Only isolate tests for unsafe mutations."]
@@ -113,6 +178,7 @@ fn main() {
             .arg(clap::arg!(--"no-build" "Do not compile the generated mutation test harness."))
             .arg(clap::arg!(--"no-run" "Do not run the generated mutation test harness."))
             .arg(clap::arg!(--color [WHEN] "Output coloring."))
+            .arg(clap::arg!(Z: -Z [FLAG] "Experimental, unstable flags. See `-Z help` for details.").action(clap::ArgAction::Append))
             // NOTE: Whenever these change, the `strip_args` calls in `run_cargo_with_mutest_driver` have to be updated as well.
             .next_help_heading("Evaluation Options")
             .arg(clap::arg!(-i --inspect [INSPECT_OPTS] "Inspect mutations after evaluation. Options may be specified in the form `--inspect=[open][:<PORT>]`.").num_args(0..=1).require_equals(true).conflicts_with_all(["no-run", "no-build", "no-emit-metadata"]))
@@ -124,8 +190,6 @@ fn main() {
             .arg(clap::arg!(--"use-thread-pool" "Evaluate tests in a fixed-size thread pool.").conflicts_with_all(["no-run", "no-build"]))
             // Printing-related Arguments
             .arg(clap::arg!(--"eval-print" [PRINT] "Print additional information during mutation evaluation. Multiple may be specified, separated by commas.").value_delimiter(',').value_parser(run_print::possible_values()).conflicts_with_all(["no-run", "no-build"]))
-            // Experimental Flags
-            .arg(clap::arg!(--"Zwrite-json-eval-stream" "Write JSONL stream file into JSON output directory specified by `--metadata-out-root-dir`.").conflicts_with_all(["no-run", "no-build", "no-emit-metadata"]))
             // Passed arguments
             .arg(clap::arg!([PASSED_OPTIONS] ...).last(true).conflicts_with_all(["no-run", "no-build"]))
             // Cargo options
@@ -168,6 +232,16 @@ fn main() {
 
     match matches.subcommand() {
         Some(("run", matches)) => {
+            let unstable_flags = matches.get_many::<String>("Z").into_iter().flatten().map(String::as_str).collect::<Vec<_>>();
+            if unstable_flags.contains(&"help") {
+                mutest_driver_cli::print_unstable_flags_help(RUN_UNSTABLE_FLAGS);
+                process::exit(0);
+            }
+            mutest_driver_cli::check_unstable_flags(&unstable_flags, RUN_UNSTABLE_FLAGS);
+            if !unstable_flags.contains(&"unstable-options") {
+                mutest_driver_cli::check_unstable_options(matches, RUN_UNSTABLE_OPTIONS);
+            }
+
             let inspect_opts = match matches.value_source("inspect") {
                 Some(clap::parser::ValueSource::CommandLine) if let Some(inspect_opts_str) = matches.get_one::<String>("inspect") => {
                     let mut unparsed_str: &str = inspect_opts_str;
@@ -211,7 +285,7 @@ fn main() {
             //       whether specified explicitly through `--target-dir`, or implicitly chosen by Cargo.
             cargo_invocation.target_dir.push("mutest");
 
-            run_cargo_with_mutest_driver(&cargo_invocation, matches);
+            run_cargo_with_mutest_driver(&cargo_invocation, matches, &unstable_flags);
 
             if let Some((open, port)) = inspect_opts {
                 run_mutest_inspector_from_cargo_invocation(open, port, &cargo_invocation, matches);
@@ -360,7 +434,7 @@ fn process_cargo_args<'a>(args: &'a [String], matches: &'a clap::ArgMatches) ->
     CargoInvocation { cargo_args, non_cargo_args, target_dir, explicit_targetings_count }
 }
 
-fn run_cargo_with_mutest_driver(cargo_invocation: &CargoInvocation, matches: &clap::ArgMatches) {
+fn run_cargo_with_mutest_driver(cargo_invocation: &CargoInvocation, matches: &clap::ArgMatches, unstable_flags: &[&str]) {
     let mut mutest_args = cargo_invocation.non_cargo_args.clone();
 
     let no_run = matches.get_flag("no-run");
@@ -380,7 +454,7 @@ fn run_cargo_with_mutest_driver(cargo_invocation: &CargoInvocation, matches: &cl
 
     let mut cmd = cargo_command_base();
 
-    let embedded = matches.get_flag("Zembedded");
+    let embedded = unstable_flags.contains(&"embedded");
     if embedded {
         let target = match matches.get_one::<String>("target") {
             Some(target) => target.clone(),
@@ -487,7 +561,7 @@ fn run_cargo_with_mutest_driver(cargo_invocation: &CargoInvocation, matches: &cl
     strip_arg(&mut mutest_args, true, None, Some("isolate"));
     strip_arg(&mut mutest_args, false, None, Some("use-thread-pool"));
     strip_arg(&mut mutest_args, true, None, Some("eval-print"));
-    strip_arg(&mut mutest_args, false, None, Some("Zwrite-json-eval-stream"));
+    strip_arg_value_occurrences(&mut mutest_args, Some("Z"), None, "write-json-eval-stream");
 
     cmd.env("MUTEST_ENCODED_ARGS", mutest_args.join("\x1F"));
 
@@ -511,8 +585,6 @@ fn run_cargo_with_mutest_driver(cargo_invocation: &CargoInvocation, matches: &cl
         if print_names.contains("all") { print_names = HashSet::from_iter(run_print::ALL.into_iter().map(|s| *s)); }
         for print_name in print_names { cmd.arg(format!("--print={print_name}")); }
 
-        if matches.get_flag("Zwrite-json-eval-stream") { cmd.arg("--Zwrite-json-eval-stream"); }
-
         cmd.args((0..matches.get_count("verbose")).map(|_| "-v"));
         if matches.get_flag("timings") { cmd.arg("--timings"); }
         if !matches.get_flag("no-emit-metadata") {
@@ -525,6 +597,8 @@ fn run_cargo_with_mutest_driver(cargo_invocation: &CargoInvocation, matches: &cl
             cmd.arg(format!("--metadata-out-root-dir={out_dir}"));
         }
 
+        if unstable_flags.contains(&"write-json-eval-stream") { cmd.arg("--Zwrite-json-eval-stream"); }
+
         cmd.args(matches.get_many::<String>("PASSED_OPTIONS").unwrap_or_default());
 
         // NOTE: Disable insta snapshot creation for mutated program tests.

cargo-mutest/src/main.rs

@@ -14,3 +14,4 @@ edition = "2024"
 
 [dependencies]
 clap = { version = "4", features = ["cargo"] }
+color-print = "0.3"

mutest-driver-cli/Cargo.toml

@@ -1,6 +1,100 @@
 #![feature(decl_macro)]
+#![feature(if_let_guard)]
 
 use std::path::PathBuf;
+use std::process;
+
+pub macro extend_const_slice($base:path: &[$ty:ty], $ext:expr) {
+    &const {
+        const EXT: &[$ty] = $ext;
+        const LEN: usize = $base.len() + EXT.len();
+        let mut dst: [$ty; LEN] = match ($base.len(), EXT.len()) {
+            // HACK: We must generate a value of type `[$ty; LEN]` here
+            //       regardless of whether it will be used during const evaluation
+            //       (i.e., even if `LEN != 0`).
+            // SAFETY: For the only case this will be used (`LEN == 0`),
+            //         the value will always be a `[_; 0]`, i.e., an empty `[]`.
+            (0, 0) => unsafe { std::mem::transmute([0u8; std::mem::size_of::<$ty>() * LEN]) }
+            (0, _) => [EXT[0]; LEN],
+            _ => [$base[0]; LEN],
+        };
+        let mut i = 0;
+        while i < $base.len() {
+            dst[i] = $base[i];
+            i += 1;
+        }
+        i = 0;
+        while i < EXT.len() {
+            dst[i + $base.len()] = EXT[i];
+            i += 1;
+        }
+        dst
+    }
+}
+
+#[derive(Copy, Clone, Debug)]
+pub struct UnstableFlag {
+    pub name: &'static str,
+    pub help: Option<&'static str>,
+}
+
+impl UnstableFlag {
+    pub const fn new(name: &'static str, help: Option<&'static str>) -> Self {
+        Self { name, help }
+    }
+}
+
+pub fn print_unstable_flags_help(flags: &[UnstableFlag]) {
+    color_print::cprintln!("<bright-green,bold>Unstable Flags:</>");
+    let w_name = flags.iter().map(|flag| flag.name.len()).max().unwrap_or_default();
+    color_print::cprintln!("  <bright-blue,bold>-Z {:<w_name$}</>  Print help information.", "help");
+    for flag in flags {
+        color_print::cprint!("  <bright-blue,bold>-Z {:<w_name$}</>", flag.name);
+        if let Some(help) = &flag.help {
+            color_print::cprint!("  {}", help);
+        }
+        color_print::cprintln!("");
+    }
+}
+
+pub fn check_unstable_flags(provided_flags: &[&str], known_flags: &[UnstableFlag]) {
+    for flag in provided_flags {
+        if !known_flags.iter().any(|f| f.name == *flag) {
+            color_print::ceprintln!("<red,bold>error</>: unknown unstable flag `{}`", flag);
+            process::exit(1);
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug)]
+pub struct UnstableOption {
+    pub name: &'static str,
+    pub value: Option<&'static str>,
+}
+
+impl UnstableOption {
+    pub const fn new(name: &'static str, value: Option<&'static str>) -> Self {
+        Self { name, value }
+    }
+}
+
+pub fn check_unstable_options(matches: &clap::ArgMatches, options: &[UnstableOption]) {
+    for option in options {
+        if let None | Some(clap::parser::ValueSource::DefaultValue) = matches.value_source(option.name) { continue; }
+
+        match option.value {
+            None => {
+                color_print::ceprintln!("<red,bold>error</>: the `--{}` flag is unstable, pass `-Z unstable-options` to enable it", option.name);
+                process::exit(1);
+            }
+            Some(value) if let Some(mut values) = matches.get_many::<String>(option.name) && values.any(|v| v == value) => {
+                color_print::ceprintln!("<red,bold>error</>: the `--{}={}` option is unstable, pass `-Z unstable-options` to enable it", option.name, value);
+                process::exit(1);
+            }
+            _ => {}
+        }
+    }
+}
 
 pub macro opts(
     $all:ident, $possible_values_vis:vis $possible_values:ident where
@@ -28,6 +122,20 @@ pub macro exclusive_opts(
     }
 }
 
+pub const UNSTABLE_FLAGS: &[UnstableFlag] = &[
+    UnstableFlag::new("unstable-options", Some("Enable the use of unstable options.")),
+    // Permanently unstable options.
+    UnstableFlag::new("verify-ast-lowering", Some("Verify whether all AST nodes are mapped to their HIR counterparts.")),
+    // Experimental options.
+    UnstableFlag::new("embedded", Some("Enable experimental support for embedded-test tests and embedded firmware generation with no_std support using a tethered embedded mutation runtime.")),
+    // Legacy options.
+    UnstableFlag::new("no-sanitize-macro-expns", Some("Skip sanitizing the identifiers and paths in macro expansions. This is legacy behavior and is not recommended.")),
+];
+
+pub const UNSTABLE_OPTIONS: &[UnstableOption] = &[
+    // No unstable options at the moment.
+];
+
 pub mod mutation_operators {
     crate::opts! { ALL, pub(crate) possible_values where
         ARG_DEFAULT_SHADOW = "arg_default_shadow";
@@ -95,12 +203,6 @@ pub mod call_graph_non_local_call_view {
     }
 }
 
-pub mod verify {
-    crate::opts! { ALL, pub(crate) possible_values where
-        AST_LOWERING = "ast_lowering";
-    }
-}
-
 pub const fn rustc_version_str() -> &'static str {
     env!("RUSTC_VERSION_STR")
 }
@@ -159,11 +261,7 @@ pub fn command(name: &'static str) -> clap::Command {
         .arg(clap::arg!(--"mutant-batch-size" [MUTANT_BATCH_SIZE] "Maximum number of mutations to batch into a single batch.").default_value("1").value_parser(clap::value_parser!(usize)))
         .arg(clap::arg!(--"mutant-batch-seed" [MUTANT_BATCH_SEED] "Random seed to use for randomness during mutation batching."))
         .arg(clap::arg!(--"mutant-batch-greedy-ordering-heuristic" [MUTANT_BATCH_GREEDY_ORDERING_HEURISTIC] "Ordering heuristic to use for `greedy` mutation batching algorithm.").value_parser(mutant_batch_greedy_ordering_heuristic::possible_values()).default_value(mutant_batch_greedy_ordering_heuristic::REVERSE_CONFLICTS))
-        .arg(clap::arg!(--"mutant-batch-greedy-epsilon" [MUTANT_BATCH_GREEDY_EPSILON] "Optional epsilon parameter for `greedy` mutation batching algorithm, used to control the probability of random mutation assignment.").default_value("0").value_parser(clap::value_parser!(f64)))
-        .next_help_heading("Experimental Options")
-        .arg(clap::arg!(--Zverify [VERIFY] "Perform additional checks to verify correctness and completeness. Multiple may be specified, separated by commas.").value_delimiter(',').value_parser(verify::possible_values()))
-        .arg(clap::arg!(--Zembedded "Enable experimental support for embedded-test tests and embedded firmware generation with no_std support using a tethered embedded mutation runtime."))
-        .arg(clap::arg!(--"Zno-sanitize-macro-expns" "Skip sanitizing the identifiers and paths in the expanded output of macro invocations. This was the previous behavior and is not recommended."));
+        .arg(clap::arg!(--"mutant-batch-greedy-epsilon" [MUTANT_BATCH_GREEDY_EPSILON] "Optional epsilon parameter for `greedy` mutation batching algorithm, used to control the probability of random mutation assignment.").default_value("0").value_parser(clap::value_parser!(f64)));
 
     cmd
 }

mutest-driver-cli/src/lib.rs

@@ -155,11 +155,6 @@ pub enum MutationParallelism {
     DynamicScheduling,
 }
 
-#[derive(Clone, Debug)]
-pub struct VerifyOptions {
-    pub ast_lowering: bool,
-}
-
 #[derive(Clone, Debug)]
 pub enum MutationFilter {
     /// `file:path/to/src.rs` or `file:path/to/src.rs:15` or `file:path/to/src.rs:15:17`.
@@ -168,6 +163,13 @@ pub enum MutationFilter {
     Def(String),
 }
 
+#[derive(Clone, Debug, Default)]
+pub struct UnstableFlags {
+    pub verify_ast_lowering: bool,
+    pub embedded: bool,
+    pub no_sanitize_macro_expns: bool,
+}
+
 pub struct Options<'op, 'm> {
     pub crate_kind: CrateKind,
     pub cargo_target_kind: Option<CargoTargetKind>,
@@ -185,9 +187,7 @@ pub struct Options<'op, 'm> {
     pub mutation_filters: Vec<MutationFilter>,
     pub mutation_parallelism: Option<MutationParallelism>,
 
-    pub verify_opts: VerifyOptions,
-    pub embedded: bool,
-    pub sanitize_macro_expns: bool,
+    pub unstable_flags: UnstableFlags,
 }
 
 pub struct Config<'op, 'm> {