core: fix double encoding of entities

Author: thecodrr

packages/core/__tests__/markdown.test.ts

@@ -123,7 +123,7 @@ function google() {
 <co-co>hello</co-co>
 \`\`\``)
   ).toBe(
-    `<pre class="language-javascript"><code class="language-javascript">function google() {<br/>}<br/><br/>&lt;html&gt;good&lt;/html&gt;<br/>&lt;COmponent&gt;{() =&gt; {}}&lt;/Component&gt;<br/>&lt;co-co&gt;hello&lt;/co-co&gt;<br/></code></pre>`
+    `<pre class="language-javascript"><code class="language-javascript">function google() {<br/>}<br/><br/>&lt;html>good&lt;/html><br/>&lt;COmponent>{() => {}}&lt;/Component><br/>&lt;co-co>hello&lt;/co-co><br/></code></pre>`
   );
 });
 
@@ -175,3 +175,19 @@ test("parse highlight", (t) => {
     `<p>I am <span style="background-color: rgb(255, 255, 0);">highlighted</span> <strong>again</strong>.</p>`
   );
 });
+
+test("do not double encode html entities", async (t) => {
+  t.expect(
+    markdowntoHTML(`I am some code with entities:
+
+\`\`\`
+I am damn happy to here & and < > what else.
+\`\`\`
+
+And **something** else.`)
+  ).toBe(
+    `<p>I am some code with entities:</p>
+<pre><code>I am damn happy to here &#38; and &lt; > what else.<br/></code></pre>
+<p>And <strong>something</strong> else.</p>`
+  );
+});

packages/core/patches/hast-util-to-html+8.0.4.patch

@@ -49,15 +49,18 @@ export function markdowntoHTML(
     .use(removeComments)
     .use(convertFileEmbeds)
     .use(remarkRehype, { allowDangerousHtml: options.allowDangerousHtml })
-    .use(escapeCode, { encodeHtmlEntities: options.encodeHtmlEntities })
+    .use(escapeCode)
     .use(fixChecklistClasses)
     .use(liftLanguageToPreFromCode)
     .use(collapseMultilineParagraphs)
     .use(rehypeStringify, {
       allowDangerousHtml: options.allowDangerousHtml,
       tightSelfClosing: true,
       closeSelfClosing: true,
-      closeEmptyElements: true
+      closeEmptyElements: true,
+      characterReferences: {
+        useShortestReferences: true
+      }
     })
     .processSync(src);
   return result.value as string;
@@ -188,32 +191,31 @@ const remarkHighlight: () => Transformer = () => (tree) => {
     const children: Node<Data>[] = values.map((str, i) =>
       i % 2 === 0
         ? {
-          type: "text",
-          value: str
-        }
+            type: "text",
+            value: str
+          }
         : {
-          type: "highlight",
-          data: {
-            hName: "span",
-            hProperties: {
-              style: `background-color: rgb(255, 255, 0);`
-            }
-          },
-          children: [
-            {
-              type: "text",
-              value: str
-            }
-          ]
-        }
+            type: "highlight",
+            data: {
+              hName: "span",
+              hProperties: {
+                style: `background-color: rgb(255, 255, 0);`
+              }
+            },
+            children: [
+              {
+                type: "text",
+                value: str
+              }
+            ]
+          }
     );
     (parent as Parent).children.splice(index!, 1, ...children);
     return ["skip", index];
   });
 };
 
-const escapeCode: Plugin<[{ encodeHtmlEntities?: boolean }?], HastRoot, HastRoot> = function (options = {}) {
-  const { encodeHtmlEntities = true } = options;
+const escapeCode: Plugin<[], HastRoot, HastRoot> = function () {
   return (tree: HastRoot) => {
     visit(tree, "element", (node) => {
       if (!isElement(node, "code")) return;
@@ -224,7 +226,7 @@ const escapeCode: Plugin<[{ encodeHtmlEntities?: boolean }?], HastRoot, HastRoot
           lines.forEach((line, i) => {
             children.push({
               type: "text",
-              value: encodeHtmlEntities ? encodeNonAsciiHTML(line) : line
+              value: line
             });
             if (i !== lines.length - 1)
               children.push({