Fix issue with double decoding HTML when outputting Markdown

Author: zachleat

src/Importer.js

@@ -255,6 +255,39 @@ class Importer {
 		}
 	}
 
+	async getTransformedContent(entry, isWritingToMarkdown) {
+		let content = entry.content;
+
+		if(Importer.isHtml(entry)) {
+			let transformedHtml = content;
+			if(!isWritingToMarkdown) {
+				// decoding built-in with Markdown
+				transformedHtml = entities.decodeHTML(content);
+			}
+
+			if(!this.shouldDownloadAssets()) {
+				content = transformedHtml;
+			} else {
+				content = await this.htmlTransformer.transform(transformedHtml, entry);
+			}
+		}
+
+		if(isWritingToMarkdown) {
+			if(Importer.isText(entry)) {
+				// _only_ decode newlines
+				content = content.split("
").join("\n");
+			}
+
+			if(Importer.shouldConvertToMarkdown(entry)) {
+				await this.markdownService.asyncInit();
+
+				content = await this.markdownService.toMarkdown(content, entry);
+			}
+		}
+
+		return content;
+	}
+
 	async getEntries(options = {}) {
 		let isWritingToMarkdown = options.contentType === "markdown";
 
@@ -279,28 +312,10 @@ class Importer {
 		let promises = await Promise.allSettled(entries.map(async entry => {
 			await this.fetchRelatedMedia(entry);
 
-			if(Importer.isHtml(entry)) {
-				let decodedHtml = entities.decodeHTML(entry.content);
-				if(!this.shouldDownloadAssets()) {
-					entry.content = decodedHtml;
-				} else {
-					entry.content = await this.htmlTransformer.transform(decodedHtml, entry);
-				}
-			}
-
-			if(isWritingToMarkdown) {
-				if(Importer.isText(entry)) {
-					// _only_ decode newlines
-					entry.content = entry.content.split("
").join("\n");
-				}
-
-				if(Importer.shouldConvertToMarkdown(entry)) {
-					await this.markdownService.asyncInit();
+			entry.content = await this.getTransformedContent(entry, isWritingToMarkdown);
 
-					entry.content = await this.markdownService.toMarkdown(entry.content, entry);
-
-					entry.contentType = "markdown";
-				}
+			if(isWritingToMarkdown && Importer.shouldConvertToMarkdown(entry)) {
+				entry.contentType = "markdown";
 			}
 
 			return entry;

src/MarkdownToHtml.js

@@ -187,24 +187,26 @@ class MarkdownToHtml {
 						finalLanguage = WORDPRESS_TO_PRISM_LANGUAGE_TRANSLATION[language] || language;
 
 						// TODO customizable
-						// Questionable default: for code blocks bookended with ` (unnecessarily)
+						// Questionable default: for code blocks unnecessarily bookended with `
 						let trimmed = content.trim();
 						if(trimmed.startsWith("`") && trimmed.endsWith("`")) {
 							content = trimmed.slice(1, -1);
 						}
 					}
 
 					try {
-						if(isFromWordPress && language === "markup" && !content.trimStart().startsWith("<") || !this.prettierLanguages[finalLanguage]) {
-							// Mislabeled as "markup" (hi WordPress) or no-parser found for prettier
-							content = entities.decodeHTML(striptags(""+node.outerHTML));
-						} else if (this.prettierLanguages[finalLanguage]) {
+						if(isFromWordPress && language === "markup" && !content.trimStart().startsWith("<")) {
+							// This code block was mislabeled as "markup" (hi WordPress), so we do nothing
+						} else if(this.prettierLanguages[finalLanguage]) {
 							// Attempt to format the code with Prettier
 							let parserName = this.prettierLanguages[finalLanguage][0];
 							content = prettierSync.format(content, { parser: parserName });
+						} else {
+							// preserve \n
+							content = entities.decodeHTML(striptags(""+node.innerHTML));
 						}
 					} catch(e) {
-						console.error(`Error running code formatting on code block from ${filePath}. Returning unformatted code.`, e);
+						console.error(`Error running code formatting on code block from ${filePath}${language ? ` (${language})` : ""}. Returning unformatted code:\n\n${content}`, e);
 					}
 
 					return MarkdownToHtml.outputMarkdownCodeBlock(content, finalLanguage);

test/test.js

@@ -109,7 +109,7 @@ test("WordPress import", async (t) => {
 
 	assert.equal(cleanContent(post.content), `We’re so close to launching version 6, and we figured it was high time to make an official announcement. So, save the date for February. Font Awesome 6 will go beyond pure icon-imagination!
 
-![](assets/image-calendar-exclamation-2-eKNZqhhuChge.png)
+![](assets/image-calendar-exclamation-2-fgQFmKKbUClc.png)
 
 Save the date! February 2022 is just around the corner!
 
@@ -121,23 +121,23 @@ So, what’s new?
 
 Font Awesome 6 contains over 7,000 new icons, so you’re sure to find what you need for your project. Plus, we’ve redesigned most of our icons from scratch, so they’re more consistent and easier to use.
 
-![](assets/image-icons-2-66KjmgCOuZQw.png)
+![](assets/image-icons-2-QjKY2bnBAg1T.png)
 
 * * *
 
 ## More Styles
 
 Font Awesome 6 includes five icons styles: solid, regular, light, duotone, and the new THIN style — not to mention all of our brand icons. And coming later in 2022 is the entirely new SHARP family of styles.
 
-![](assets/image-styles-2-SNjQOsXaJuRQ.png)
+![](assets/image-styles-2-L9iIR9SlzG26.png)
 
 * * *
 
 ## More Ways to Use
 
 Font Awesome 6 makes it even easier to use icons where you want to. More plugins and packages to match your stack. Less time wrestling browser rendering.
 
-![](assets/image-awesome-2-1AOLfzrlbkMJ.png)
+![](assets/image-awesome-2-O2xta1tL8p5n.png)
 
 * * *
 
@@ -252,3 +252,32 @@ test("Fetcher asset location tests (absolute)", async (t) => {
 		url: "/assets/test-NzhbK6MSYu2g.png",
 	});
 });
+
+
+test("Markdown cleanup code blocks strip tags", async (t) => {
+	let importer = new Importer();
+
+	importer.setVerbose(false);
+	importer.setDryRun(true);
+
+	let content = await importer.getTransformedContent({
+		content: `<pre id=\"b088\" class=\"graf graf--pre graf-after--p\">$ npm config set “<a class=\"markup--anchor markup--pre-anchor\" title=\"Twitter profile for @fortawesome\" href=\"http://twitter.com/fortawesome\" target=\"_blank\" rel=\"noopener noreferrer\" data-href=\"http://twitter.com/fortawesome\">@fortawesome</a>:registry” <a class=\"markup--anchor markup--pre-anchor\" href=\"https://npm.fontawesome.com/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" data-href=\"https://npm.fontawesome.com/\">https://npm.fontawesome.com/</a></pre>`,
+		contentType: "html"
+	}, true);
+
+	assert.equal(content.trim(), "```\n$ npm config set “@fortawesome:registry” https://npm.fontawesome.com/\n```");
+});
+
+test("Markdown cleanup code blocks with nested <code>", async (t) => {
+	let importer = new Importer();
+
+	importer.setVerbose(false);
+	importer.setDryRun(true);
+
+	let content2 = await importer.getTransformedContent({
+		content: `<pre id=\"9da4\" class=\"graf graf--pre graf-after--p\">Authorization: Bearer <code>DEAD-BEEF</code></pre>`,
+		contentType: "html"
+	}, true);
+
+	assert.equal(content2.trim(), "```\nAuthorization: Bearer DEAD-BEEF\n```");
+});