Merge pull request #601 from LHMQ878/fix/556-injected-tag-suffix

fix: strip injected message-id suffix before paired tag regex

Author: Tarquinen

lib/messages/utils.ts

@@ -7,7 +7,8 @@ const SUMMARY_ID_HASH_LENGTH = 16
 const DCP_BLOCK_ID_TAG_REGEX = /(<dcp-message-id(?=[\s>])[^>]*>)b\d+(<\/dcp-message-id>)/g
 const DCP_PAIRED_TAG_REGEX = /<dcp[^>]*>[\s\S]*?<\/dcp[^>]*>/gi
 const DCP_UNPAIRED_TAG_REGEX = /<\/?dcp[^>]*>/gi
-const HALLUCINATED_PARAMETER_SUFFIX_REGEX = /\nm\d+<\/parameter>\s*$/
+const INJECTED_MESSAGE_ID_SUFFIX_REGEX = /(?<=\n)<dcp-message-id[^>]*>m\d+<\/dcp-message-id>\s*$/
+const HALLUCINATED_PARAMETER_SUFFIX_REGEX = /(?<=\n)m\d+<\/parameter>\s*$/
 
 const generateStableId = (prefix: string, seed: string): string => {
     const hash = createHash("sha256").update(seed).digest("hex").slice(0, SUMMARY_ID_HASH_LENGTH)
@@ -164,8 +165,12 @@ export const replaceBlockIdsWithBlocked = (text: string): string => {
 }
 
 export const stripHallucinationsFromString = (text: string): string => {
-    const withoutHallucinatedParameter = text.replace(HALLUCINATED_PARAMETER_SUFFIX_REGEX, "")
-    return withoutHallucinatedParameter.replace(DCP_PAIRED_TAG_REGEX, "").replace(DCP_UNPAIRED_TAG_REGEX, "")
+    const withoutKnownSuffixes = text
+        .replace(INJECTED_MESSAGE_ID_SUFFIX_REGEX, "")
+        .replace(HALLUCINATED_PARAMETER_SUFFIX_REGEX, "")
+    return withoutKnownSuffixes
+        .replace(DCP_PAIRED_TAG_REGEX, "")
+        .replace(DCP_UNPAIRED_TAG_REGEX, "")
 }
 
 export const stripHallucinations = (messages: WithParts[]): void => {

package-lock.json

@@ -814,6 +814,30 @@ test("hallucination stripping does not affect non-dcp tags", async () => {
     )
 })
 
+test("hallucination stripping preserves content when dcp-message-id is mentioned in text (issue #556)", () => {
+    const input =
+        "The tag called `<dcp-message-id>` is used to track messages. " +
+        "This text should survive.\n\n" +
+        "<dcp-message-id>m0369</dcp-message-id>"
+
+    assert.equal(
+        stripHallucinationsFromString(input),
+        "The tag called `` is used to track messages. This text should survive.\n\n",
+    )
+})
+
+test("hallucination stripping handles priority on injected message-id suffixes", () => {
+    const input =
+        "The tag called `<dcp-message-id>` is used to track messages. " +
+        "This text should survive.\n\n" +
+        '<dcp-message-id priority="low">m0370</dcp-message-id>'
+
+    assert.equal(
+        stripHallucinationsFromString(input),
+        "The tag called `` is used to track messages. This text should survive.\n\n",
+    )
+})
+
 test("hallucination stripping removes trailing mXXXX</parameter> artifact (issue #555)", () => {
     assert.equal(
         stripHallucinationsFromString("Total: maybe 20 lines changed.\n\nm0340</parameter>\n\n"),