Replace hand-written markdown normalization with proper stripping

parser/src/lib.rs

@@ -2,7 +2,9 @@ pub mod command;
 pub mod error;
 mod ignore_block;
 mod mentions;
+mod text;
 mod token;
 
 pub use ignore_block::replace_all_outside_ignore_blocks;
 pub use mentions::get_mentions;
+pub use text::strip_markdown;

parser/src/text.rs

@@ -0,0 +1,83 @@
+use pulldown_cmark::{Event, Options, Parser, TagEnd};
+
+pub fn strip_markdown(text: &str) -> String {
+    let mut buffer = String::new();
+
+    let mut parser = Parser::new_ext(
+        text,
+        Options::ENABLE_TABLES
+            | Options::ENABLE_GFM
+            | Options::ENABLE_MATH
+            | Options::ENABLE_STRIKETHROUGH
+            | Options::ENABLE_FOOTNOTES
+            | Options::ENABLE_HEADING_ATTRIBUTES,
+    )
+    .into_iter();
+
+    while let Some(event) = parser.next() {
+        match event {
+            // Text and inline code are the content we want to keep
+            Event::Text(t) | Event::Code(t) => {
+                let stripped = t.replace("@", ""); // Strip mentions as well
+                buffer.push_str(&stripped);
+            }
+            // Add a newline when a block-level element ends to maintain spacing
+            Event::End(tag) => {
+                if is_block_tag(&tag) {
+                    buffer.push('\n');
+                }
+            }
+            _ => {}
+        }
+    }
+
+    buffer.trim().to_string()
+}
+
+fn is_block_tag(tag: &TagEnd) -> bool {
+    matches!(
+        tag,
+        TagEnd::Paragraph | TagEnd::Heading { .. } | TagEnd::Item | TagEnd::CodeBlock
+    )
+}
+
+#[test]
+fn basic_formatting() {
+    let input = "This is **bold**, *italic* and ~~strikethrough~~ text.";
+    let expected = "This is bold, italic and strikethrough text.";
+    assert_eq!(strip_markdown(input), expected);
+}
+
+#[test]
+fn links_and_images() {
+    let input = "Check out [Google](https://google.com) and this ![alt text](image.png).";
+    let expected = "Check out Google and this alt text.";
+    assert_eq!(strip_markdown(input), expected);
+}
+
+#[test]
+fn headers_and_lists() {
+    let input = "# Title\n- Item 1\n- Item 2";
+    let expected = "Title\nItem 1\nItem 2";
+    assert_eq!(strip_markdown(input), expected);
+}
+
+#[test]
+fn inline_code() {
+    let input = "Use the `fn main()` function.";
+    let expected = "Use the fn main() function.";
+    assert_eq!(strip_markdown(input), expected);
+}
+
+#[test]
+fn test_pings() {
+    let input = "let's ping @foo";
+    let expected = "let's ping foo";
+    assert_eq!(strip_markdown(input), expected);
+}
+
+#[test]
+fn empty_and_whitespace() {
+    assert_eq!(strip_markdown(""), "");
+    assert_eq!(strip_markdown("   "), "");
+}

src/zulip.rs

@@ -1658,14 +1658,7 @@ fn format_date(date: Option<DateTime<Utc>>) -> String {
 /// Truncates the given text to the specified length, adding ellipsis if needed.
 /// Also removes various special Markdown symbols from it.
 fn truncate_and_normalize(text: &str, max_len: usize) -> String {
-    let normalized: String = text
-        .split_whitespace()
-        .collect::<Vec<_>>()
-        .join(" ")
-        // Avoid things that could interact with the rest of the Markdown message.
-        // And avoid and @ pings (in case the text is outputted in a public Zulip stream).
-        .replace("`", "")
-        .replace("@", "");
+    let normalized = parser::strip_markdown(text);
 
     if normalized.len() <= max_len {
         normalized