fix: recognize code-fence language when the directive uses a comma (#806)

fix: recognize language when code-fence directive uses a comma

A code block whose info string separates the language from a directive with
a comma (e.g. ```ts, no-eval```) was not syntax-highlighted. Markdown parsers
split the info string at the first whitespace, so the language token arrived
as `ts,` with the comma attached, which failed the example-language check and
was passed verbatim to the syntax highlighter.

Add a `language_token` helper that strips directive separators (whitespace and
commas) and use it both when processing hidden example lines and when handing
the language to the highlighter, so `ts,`, `ts, no-eval` and `ts,no-eval` are
all treated as `ts`.

Author: crowlbot

src/html/comrak.rs

@@ -212,6 +212,12 @@ impl SyntaxHighlighterAdapter for ComrakHighlightWrapperAdapter {
     lang: Option<&str>,
     code: &str,
   ) -> std::io::Result<()> {
+    // The markdown parser hands us the info string up to the first whitespace,
+    // so a comma-separated directive such as `ts, no-eval` arrives as `ts,`.
+    // Normalize it to the bare language token so both hidden-line processing and
+    // the downstream highlighter recognize it.
+    let lang = lang.map(crate::util::example_code::language_token);
+
     // Resolve any "hidden" lines (rustdoc style): they are removed from the
     // displayed code but kept in the copyable form so copied snippets still
     // run. Non-example languages are left untouched.
@@ -303,4 +309,17 @@ mod tests {
     assert!(displayed.contains("# shown"));
     assert!(!displayed.contains("## shown"));
   }
+
+  #[test]
+  fn comma_in_info_string_is_treated_as_example() {
+    // A comma-separated directive (`ts, no-eval`) leaves the parser handing us
+    // `ts,`; it must still be recognized as a `ts` example so hidden lines are
+    // processed (regression test for broken highlighting on commas).
+    let html = render("```ts, no-eval\n# const x = 1;\nconsole.log(x);\n```");
+    let displayed = html.split("</code>").next().unwrap();
+    assert!(!displayed.contains("const x = 1;"));
+    assert!(displayed.contains("console.log(x);"));
+    // The full snippet is still copyable.
+    assert!(html.contains("const x = 1;"));
+  }
 }

src/util/example_code.rs

@@ -12,6 +12,21 @@
 //!
 //! See <https://doc.rust-lang.org/rustdoc/write-documentation/documentation-tests.html#hiding-portions-of-the-example>.
 
+/// Extracts the language token from a fenced code block's info string.
+///
+/// The info string may carry additional attributes after the language, written
+/// either space-separated (`ts ignore`) or comma-separated (`ts, no-eval` /
+/// `ts,no-eval`). Only the leading token denotes the language. Markdown parsers
+/// typically split the info string at the first whitespace, which leaves a
+/// trailing comma attached (`ts,`); splitting on commas as well as whitespace
+/// recovers the bare language (`ts`).
+pub fn language_token(lang: &str) -> &str {
+  lang
+    .split(|c: char| c.is_whitespace() || c == ',')
+    .next()
+    .unwrap_or("")
+}
+
 /// Whether a fenced code block's language denotes a runnable JavaScript or
 /// TypeScript example.
 ///
@@ -53,9 +68,7 @@ pub fn process_example_code(
   lang: Option<&str>,
   code: &str,
 ) -> Option<ExampleCode> {
-  // The info string may carry additional attributes (e.g. `ts ignore`); only
-  // the first token denotes the language.
-  let lang = lang.unwrap_or("").split_whitespace().next().unwrap_or("");
+  let lang = language_token(lang.unwrap_or(""));
   if !is_example_lang(lang) {
     return None;
   }
@@ -187,6 +200,27 @@ mod tests {
     assert_eq!(displayed(Some("ts ignore"), code).unwrap(), "shown;\n");
   }
 
+  #[test]
+  fn language_token_strips_directive_separators() {
+    assert_eq!(language_token("ts"), "ts");
+    assert_eq!(language_token("ts no-eval"), "ts");
+    // A comma-separated directive (markdown parsers leave the comma attached to
+    // the first token, so this arrives as `ts,`).
+    assert_eq!(language_token("ts,"), "ts");
+    assert_eq!(language_token("ts, no-eval"), "ts");
+    assert_eq!(language_token("ts,no-eval"), "ts");
+    assert_eq!(language_token(""), "");
+  }
+
+  #[test]
+  fn handles_comma_separated_info_string() {
+    // Regression test: a comma in the directive (`ts, no-eval`) must not stop
+    // the language from being recognized as a runnable example.
+    let code = "# hidden;\nshown;\n";
+    assert_eq!(displayed(Some("ts,"), code).unwrap(), "shown;\n");
+    assert_eq!(displayed(Some("ts, no-eval"), code).unwrap(), "shown;\n");
+  }
+
   #[test]
   fn handles_missing_trailing_newline() {
     let code = "# hidden;\nshown;";