compiletest: Report filtered and --skip-ed tests in metrics

jyn514 · jyn514 · commit b35ca65ae999 · 2026-04-20T17:17:27.000+02:00
diff --git a/src/bootstrap/src/utils/render_tests.rs b/src/bootstrap/src/utils/render_tests.rs
@@ -155,7 +155,9 @@ impl<'a> Renderer<'a> {
     }
 
     fn render_test_outcome(&mut self, outcome: Outcome<'_>, test: &TestOutcome) {
-        self.executed_tests += 1;
+        if !matches!(outcome, Outcome::FilteredOut) {
+            self.executed_tests += 1;
+        }
 
         if let Outcome::Ignored { reason } = outcome {
             self.ignored_tests += 1;
@@ -171,6 +173,7 @@ impl<'a> Renderer<'a> {
             match outcome {
                 Outcome::Ok | Outcome::BenchOk => build_helper::metrics::TestOutcome::Passed,
                 Outcome::Failed => build_helper::metrics::TestOutcome::Failed,
+                Outcome::FilteredOut => build_helper::metrics::TestOutcome::FilteredOut,
                 Outcome::Ignored { reason } => build_helper::metrics::TestOutcome::Ignored {
                     ignore_reason: reason.map(|s| s.to_string()),
                 },
@@ -209,7 +212,9 @@ impl<'a> Renderer<'a> {
             self.terse_tests_in_line = 0;
         }
 
-        self.terse_tests_in_line += 1;
+        if !matches!(outcome, Outcome::FilteredOut) {
+            self.terse_tests_in_line += 1;
+        }
         self.builder.colored_stdout(|stdout| outcome.write_short(stdout, &test.name)).unwrap();
         let _ = std::io::stdout().flush();
     }
@@ -359,6 +364,9 @@ impl<'a> Renderer<'a> {
                     &outcome,
                 );
             }
+            Message::Test(TestMessage::FilteredOut(outcome)) => {
+                self.render_test_outcome(Outcome::FilteredOut, &outcome);
+            }
             Message::Test(TestMessage::Failed(outcome)) => {
                 self.render_test_outcome(Outcome::Failed, &outcome);
                 self.failures.push(outcome);
@@ -371,11 +379,13 @@ impl<'a> Renderer<'a> {
     }
 }
 
+#[derive(Debug)]
 enum Outcome<'a> {
     Ok,
     BenchOk,
     Failed,
     Ignored { reason: Option<&'a str> },
+    FilteredOut,
 }
 
 impl Outcome<'_> {
@@ -400,6 +410,7 @@ impl Outcome<'_> {
                 writer.set_color(ColorSpec::new().set_fg(Some(Color::Yellow)))?;
                 write!(writer, "i")?;
             }
+            Outcome::FilteredOut => {}
         }
         writer.reset()
     }
@@ -425,13 +436,14 @@ impl Outcome<'_> {
                     write!(writer, ", {reason}")?;
                 }
             }
+            Outcome::FilteredOut => {}
         }
         writer.reset()
     }
 
     fn write_ci(&self, writer: &mut dyn WriteColor, name: &str) -> Result<(), std::io::Error> {
         match self {
-            Outcome::Ok | Outcome::BenchOk | Outcome::Ignored { .. } => {}
+            Outcome::Ok | Outcome::BenchOk | Outcome::FilteredOut | Outcome::Ignored { .. } => {}
             Outcome::Failed => {
                 writer.set_color(ColorSpec::new().set_fg(Some(Color::Red)))?;
                 writeln!(writer, "   {name} ... FAILED")?;
diff --git a/src/build_helper/src/metrics.rs b/src/build_helper/src/metrics.rs
@@ -38,6 +38,7 @@ pub mod compiletest {
         Ok(TestOutcome),
         Failed(TestOutcome),
         Ignored(TestOutcome),
+        FilteredOut(TestOutcome),
         Timeout { name: String },
         Started,
     }
@@ -155,6 +156,7 @@ pub struct Test {
 pub enum TestOutcome {
     Passed,
     Failed,
+    FilteredOut,
     Ignored { ignore_reason: Option<String> },
 }
 
diff --git a/src/ci/citool/src/analysis.rs b/src/ci/citool/src/analysis.rs
@@ -155,7 +155,9 @@ fn report_debuginfo_statistics(suites: &[&TestSuite]) {
                         debugger_test_record.entry(kind).or_insert(TestSuiteRecord::default());
                     match test.outcome {
                         TestOutcome::Passed => record.passed += 1,
-                        TestOutcome::Ignored { .. } => record.ignored += 1,
+                        TestOutcome::Ignored { .. } | TestOutcome::FilteredOut => {
+                            record.ignored += 1
+                        }
                         TestOutcome::Failed => record.failed += 1,
                     }
                 }
@@ -336,7 +338,7 @@ fn aggregate_test_suites(suites: &[&TestSuite]) -> BTreeMap<String, TestSuiteRec
                 TestOutcome::Failed => {
                     record.failed += 1;
                 }
-                TestOutcome::Ignored { .. } => {
+                TestOutcome::Ignored { .. } | TestOutcome::FilteredOut => {
                     record.ignored += 1;
                 }
             }
@@ -470,6 +472,7 @@ fn report_test_diffs(
         match outcome {
             TestOutcome::Passed => "pass".to_string(),
             TestOutcome::Failed => "fail".to_string(),
+            TestOutcome::FilteredOut => "ignore".to_string(),
             TestOutcome::Ignored { ignore_reason } => {
                 let reason = match ignore_reason {
                     Some(reason) => format!(" ({reason})"),
diff --git a/src/ci/citool/src/test_dashboard.rs b/src/ci/citool/src/test_dashboard.rs
@@ -77,7 +77,7 @@ fn gather_test_suites(job_metrics: &HashMap<JobName, JobMetrics>) -> TestSuites<
                     TestOutcome::Passed => {
                         variant_entry.passed.push(test_metadata);
                     }
-                    TestOutcome::Ignored { ignore_reason: _ } => {
+                    TestOutcome::Ignored { ignore_reason: _ } | TestOutcome::FilteredOut => {
                         variant_entry.ignored.push(test_metadata);
                     }
                     TestOutcome::Failed => {
diff --git a/src/tools/compiletest/src/executor.rs b/src/tools/compiletest/src/executor.rs
@@ -23,7 +23,7 @@ mod json;
 
 pub(crate) fn run_tests(config: &Config, tests: Vec<CollectedTest>) -> bool {
     let tests_len = tests.len();
-    let filtered = filter_tests(config, tests);
+    let (filtered, ignored) = filter_tests(config, tests);
     // Iterator yielding tests that haven't been started yet.
     let mut fresh_tests = (0..).map(TestId).zip(&filtered);
 
@@ -41,6 +41,11 @@ pub(crate) fn run_tests(config: &Config, tests: Vec<CollectedTest>) -> bool {
     let num_filtered_out = tests_len - filtered.len();
     listener.suite_started(filtered.len(), num_filtered_out);
 
+    for (id, test) in (filtered.len()..).map(TestId).zip(ignored) {
+        let completion = TestCompletion { id, outcome: TestOutcome::FilteredOut, stdout: None };
+        listener.test_finished(&test, &completion);
+    }
+
     // Channel used by test threads to report the test outcome when done.
     let (completion_tx, completion_rx) = mpsc::channel::<TestCompletion>();
 
@@ -263,6 +268,7 @@ enum TestOutcome {
     Succeeded,
     Failed { message: Option<&'static str> },
     Ignored,
+    FilteredOut,
 }
 
 impl TestOutcome {
@@ -278,9 +284,10 @@ impl TestOutcome {
 /// FIXME(#139660): Now that libtest has been removed, redesign the whole filtering system to
 /// do a better job of understanding and filtering _paths_, instead of being tied to libtest's
 /// substring/exact matching behaviour.
-fn filter_tests(opts: &Config, tests: Vec<CollectedTest>) -> Vec<CollectedTest> {
-    let mut filtered = tests;
-
+fn filter_tests(
+    opts: &Config,
+    tests: Vec<CollectedTest>,
+) -> (Vec<CollectedTest>, Vec<CollectedTest>) {
     let matches_filter = |test: &CollectedTest, filter_str: &str| {
         if opts.filter_exact {
             // When `--exact` is used we must use `filterable_path` to get
@@ -293,17 +300,23 @@ fn filter_tests(opts: &Config, tests: Vec<CollectedTest>) -> Vec<CollectedTest>
         }
     };
 
-    // Remove tests that don't match the test filter
-    if !opts.filters.is_empty() {
-        filtered.retain(|test| opts.filters.iter().any(|filter| matches_filter(test, filter)));
-    }
+    let should_run = |test: &CollectedTest| {
+        // Remove tests that don't match the test filter
+        if !opts.filters.is_empty()
+            && !opts.filters.iter().any(|filter| matches_filter(test, filter))
+        {
+            return false;
+        }
 
-    // Skip tests that match any of the skip filters
-    if !opts.skip.is_empty() {
-        filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf)));
-    }
+        // Skip tests that match any of the skip filters
+        if opts.skip.iter().any(|sf| matches_filter(test, sf)) {
+            return false;
+        }
+
+        true
+    };
 
-    filtered
+    tests.into_iter().partition(should_run)
 }
 
 /// Determines the number of tests to run concurrently.
diff --git a/src/tools/compiletest/src/executor/json.rs b/src/tools/compiletest/src/executor/json.rs
@@ -71,6 +71,10 @@ impl Listener {
                 maybe_message = test.desc.ignore_message.as_deref();
                 event = "ignored";
             }
+            TestOutcome::FilteredOut => {
+                maybe_message = test.desc.ignore_message.as_deref();
+                event = "filtered_out";
+            }
         };
 
         // This emits optional fields as `null`, instead of omitting them
diff --git a/src/tools/compiletest/tests/flags.rs b/src/tools/compiletest/tests/flags.rs
@@ -96,13 +96,12 @@ impl CompiletestMetrics {
                 Message::Test(inner @ TestMessage::Timeout { name }) => (inner, name),
                 _ => return None,
             };
-            let stripped = name.strip_prefix("[ui] tests/ui/compiletest-self-test/").unwrap()
-                .strip_suffix(".rs").unwrap();
-            if stripped == expected {
-                Some(msg)
-            } else {
-                None
-            }
+            let stripped = name
+                .strip_prefix("[ui] ")
+                .unwrap_or_else(|| panic!("unknown name format {name}"))
+                // This can be None if compiletest prints ignored tests from other directories.
+                .strip_prefix("tests/ui/compiletest-self-test/");
+            if stripped == Some(expected) { Some(msg) } else { None }
         })
     }
 }
@@ -129,7 +128,18 @@ impl CompiletestMetricsExt for Assert {
 
 #[test]
 fn ignore_directive_tests_reported_ignored() {
+    let ct = compiletest();
+    let metrics = run_ok(ct).metrics();
+    assert!(matches!(metrics.find_test("ignore-directive.rs").unwrap(), TestMessage::Ignored(..)));
+}
+
+#[test]
+fn skipped_tests_reported_filtered() {
     let mut ct = compiletest();
-    let metrics = dbg!(run_ok(ct).metrics());
-    assert!(matches!(metrics.find_test("ignore-directive").unwrap(), TestMessage::Ignored(..)));
+    ct.arg("--skip=ignore-directive");
+    let metrics = run_ok(ct).metrics();
+    assert!(matches!(
+        metrics.find_test("ignore-directive.rs").unwrap(),
+        TestMessage::FilteredOut(..)
+    ));
 }

Original file line number	Diff line number	Diff line change
`@@ -155,7 +155,9 @@ impl<'a> Renderer<'a> {`
`155`	`155`	`}`
`156`	`156`
`157`	`157`	`fn render_test_outcome(&mut self, outcome: Outcome<'_>, test: &TestOutcome) {`
`158`		`- self.executed_tests += 1;`
	`158`	`+ if !matches!(outcome, Outcome::FilteredOut) {`
	`159`	`+ self.executed_tests += 1;`
	`160`	`+ }`
`159`	`161`
`160`	`162`	`if let Outcome::Ignored { reason } = outcome {`
`161`	`163`	`self.ignored_tests += 1;`
`@@ -171,6 +173,7 @@ impl<'a> Renderer<'a> {`
`171`	`173`	`match outcome {`
`172`	`174`	`Outcome::Ok \| Outcome::BenchOk => build_helper::metrics::TestOutcome::Passed,`
`173`	`175`	`Outcome::Failed => build_helper::metrics::TestOutcome::Failed,`
	`176`	`+ Outcome::FilteredOut => build_helper::metrics::TestOutcome::FilteredOut,`
`174`	`177`	`Outcome::Ignored { reason } => build_helper::metrics::TestOutcome::Ignored {`
`175`	`178`	`ignore_reason: reason.map(\|s\| s.to_string()),`
`176`	`179`	`},`
`@@ -209,7 +212,9 @@ impl<'a> Renderer<'a> {`
`209`	`212`	`self.terse_tests_in_line = 0;`
`210`	`213`	`}`
`211`	`214`
`212`		`- self.terse_tests_in_line += 1;`
	`215`	`+ if !matches!(outcome, Outcome::FilteredOut) {`
	`216`	`+ self.terse_tests_in_line += 1;`
	`217`	`+ }`
`213`	`218`	`self.builder.colored_stdout(\|stdout\| outcome.write_short(stdout, &test.name)).unwrap();`
`214`	`219`	`let _ = std::io::stdout().flush();`
`215`	`220`	`}`
`@@ -359,6 +364,9 @@ impl<'a> Renderer<'a> {`
`359`	`364`	`&outcome,`
`360`	`365`	`);`
`361`	`366`	`}`
	`367`	`+ Message::Test(TestMessage::FilteredOut(outcome)) => {`
	`368`	`+ self.render_test_outcome(Outcome::FilteredOut, &outcome);`
	`369`	`+ }`
`362`	`370`	`Message::Test(TestMessage::Failed(outcome)) => {`
`363`	`371`	`self.render_test_outcome(Outcome::Failed, &outcome);`
`364`	`372`	`self.failures.push(outcome);`
`@@ -371,11 +379,13 @@ impl<'a> Renderer<'a> {`
`371`	`379`	`}`
`372`	`380`	`}`
`373`	`381`
	`382`	`+#[derive(Debug)]`
`374`	`383`	`enum Outcome<'a> {`
`375`	`384`	`Ok,`
`376`	`385`	`BenchOk,`
`377`	`386`	`Failed,`
`378`	`387`	`Ignored { reason: Option<&'a str> },`
	`388`	`+ FilteredOut,`
`379`	`389`	`}`
`380`	`390`
`381`	`391`	`impl Outcome<'_> {`
`@@ -400,6 +410,7 @@ impl Outcome<'_> {`
`400`	`410`	`writer.set_color(ColorSpec::new().set_fg(Some(Color::Yellow)))?;`
`401`	`411`	`write!(writer, "i")?;`
`402`	`412`	`}`
	`413`	`+ Outcome::FilteredOut => {}`
`403`	`414`	`}`
`404`	`415`	`writer.reset()`
`405`	`416`	`}`
`@@ -425,13 +436,14 @@ impl Outcome<'_> {`
`425`	`436`	`write!(writer, ", {reason}")?;`
`426`	`437`	`}`
`427`	`438`	`}`
	`439`	`+ Outcome::FilteredOut => {}`
`428`	`440`	`}`
`429`	`441`	`writer.reset()`
`430`	`442`	`}`
`431`	`443`
`432`	`444`	`fn write_ci(&self, writer: &mut dyn WriteColor, name: &str) -> Result<(), std::io::Error> {`
`433`	`445`	`match self {`
`434`		`- Outcome::Ok \| Outcome::BenchOk \| Outcome::Ignored { .. } => {}`
	`446`	`+ Outcome::Ok \| Outcome::BenchOk \| Outcome::FilteredOut \| Outcome::Ignored { .. } => {}`
`435`	`447`	`Outcome::Failed => {`
`436`	`448`	`writer.set_color(ColorSpec::new().set_fg(Some(Color::Red)))?;`
`437`	`449`	`writeln!(writer, " {name} ... FAILED")?;`
Original file line number	Diff line number	Diff line change
`@@ -38,6 +38,7 @@ pub mod compiletest {`
`38`	`38`	`Ok(TestOutcome),`
`39`	`39`	`Failed(TestOutcome),`
`40`	`40`	`Ignored(TestOutcome),`
	`41`	`+ FilteredOut(TestOutcome),`
`41`	`42`	`Timeout { name: String },`
`42`	`43`	`Started,`
`43`	`44`	`}`
`@@ -155,6 +156,7 @@ pub struct Test {`
`155`	`156`	`pub enum TestOutcome {`
`156`	`157`	`Passed,`
`157`	`158`	`Failed,`
	`159`	`+ FilteredOut,`
`158`	`160`	`Ignored { ignore_reason: Option<String> },`
`159`	`161`	`}`
`160`	`162`
Original file line number	Diff line number	Diff line change
`@@ -155,7 +155,9 @@ fn report_debuginfo_statistics(suites: &[&TestSuite]) {`
`155`	`155`	`debugger_test_record.entry(kind).or_insert(TestSuiteRecord::default());`
`156`	`156`	`match test.outcome {`
`157`	`157`	`TestOutcome::Passed => record.passed += 1,`
`158`		`- TestOutcome::Ignored { .. } => record.ignored += 1,`
	`158`	`+ TestOutcome::Ignored { .. } \| TestOutcome::FilteredOut => {`
	`159`	`+ record.ignored += 1`
	`160`	`+ }`
`159`	`161`	`TestOutcome::Failed => record.failed += 1,`
`160`	`162`	`}`
`161`	`163`	`}`
`@@ -336,7 +338,7 @@ fn aggregate_test_suites(suites: &[&TestSuite]) -> BTreeMap<String, TestSuiteRec`
`336`	`338`	`TestOutcome::Failed => {`
`337`	`339`	`record.failed += 1;`
`338`	`340`	`}`
`339`		`- TestOutcome::Ignored { .. } => {`
	`341`	`+ TestOutcome::Ignored { .. } \| TestOutcome::FilteredOut => {`
`340`	`342`	`record.ignored += 1;`
`341`	`343`	`}`
`342`	`344`	`}`
`@@ -470,6 +472,7 @@ fn report_test_diffs(`
`470`	`472`	`match outcome {`
`471`	`473`	`TestOutcome::Passed => "pass".to_string(),`
`472`	`474`	`TestOutcome::Failed => "fail".to_string(),`
	`475`	`+ TestOutcome::FilteredOut => "ignore".to_string(),`
`473`	`476`	`TestOutcome::Ignored { ignore_reason } => {`
`474`	`477`	`let reason = match ignore_reason {`
`475`	`478`	`Some(reason) => format!(" ({reason})"),`
Original file line number	Diff line number	Diff line change
`@@ -77,7 +77,7 @@ fn gather_test_suites(job_metrics: &HashMap<JobName, JobMetrics>) -> TestSuites<`
`77`	`77`	`TestOutcome::Passed => {`
`78`	`78`	`variant_entry.passed.push(test_metadata);`
`79`	`79`	`}`
`80`		`- TestOutcome::Ignored { ignore_reason: _ } => {`
	`80`	`+ TestOutcome::Ignored { ignore_reason: _ } \| TestOutcome::FilteredOut => {`
`81`	`81`	`variant_entry.ignored.push(test_metadata);`
`82`	`82`	`}`
`83`	`83`	`TestOutcome::Failed => {`