Merge pull request #27 from inaridiy/feat/improve-accuracy

Feat/improve accuracy

Author: inaridiy

.changeset/honest-pens-jam.md

@@ -0,0 +1,5 @@
+---
+"webforai": patch
+---
+
+Improve extract algorithm

packages/webforai/package.json

@@ -4,15 +4,7 @@
 	"description": "A library that provides a web interface for AI",
 	"author": "inaridiy",
 	"license": "MIT",
-	"keywords": [
-		"web",
-		"ai",
-		"html",
-		"html2md",
-		"markdown",
-		"mdast",
-		"hast"
-	],
+	"keywords": ["web", "ai", "html", "html2md", "markdown", "mdast", "hast"],
 	"repository": {
 		"type": "git",
 		"url": "https://github.com/inaridiy/webforai.git"
@@ -26,10 +18,7 @@
 		"prerelease": "pnpm build",
 		"release": "np"
 	},
-	"files": [
-		"dist",
-		"!dist/types/**/*.js"
-	],
+	"files": ["dist", "!dist/types/**/*.js"],
 	"main": "dist/cjs/index.js",
 	"type": "module",
 	"module": "dist/index.js",
@@ -68,21 +57,11 @@
 	},
 	"typesVersions": {
 		"*": {
-			"types": [
-				"./dist/types/index.d.ts"
-			],
-			"loaders/playwright": [
-				"./dist/types/loaders/playwright.d.ts"
-			],
-			"loaders/cf-puppeteer": [
-				"./dist/types/loaders/cf-puppeteer.d.ts"
-			],
-			"loaders/fetch": [
-				"./dist/types/loaders/fetch.d.ts"
-			],
-			"loaders/puppeteer": [
-				"./dist/types/loaders/puppeteer.d.ts"
-			]
+			"types": ["./dist/types/index.d.ts"],
+			"loaders/playwright": ["./dist/types/loaders/playwright.d.ts"],
+			"loaders/cf-puppeteer": ["./dist/types/loaders/cf-puppeteer.d.ts"],
+			"loaders/fetch": ["./dist/types/loaders/fetch.d.ts"],
+			"loaders/puppeteer": ["./dist/types/loaders/puppeteer.d.ts"]
 		}
 	},
 	"peerDependencies": {

packages/webforai/src/extract-hast/readability.ts

@@ -2,17 +2,8 @@ import type { Element, Nodes as Hast } from "hast";
 import { select, selectAll } from "hast-util-select";
 import { toString as hastToString } from "hast-util-to-string";
 import { filter } from "unist-util-filter";
-import { parents } from "unist-util-parents";
 import { classnames, hasAncestors, isStrInclude, matchString } from "./utils";
 
-type ProxiedHast = Hast & { parent: ProxiedHast | null };
-
-declare module "hast" {
-	interface Element {
-		parent: ProxiedHast | null;
-	}
-}
-
 const UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
 
 const REGEXPS = {
@@ -159,9 +150,9 @@ const removeEmptyFilter = (node: Hast) => {
 export const readabilityExtractHast = (hast: Hast): Hast => {
 	const lang = String(select("html", hast)?.properties.lang || tryGetLang(hast) || "en");
 	const body = select("body", hast) ?? hast;
+	const bodyText = hastToString(body);
 
-	const proxiedHast = parents(body) as unknown as ProxiedHast;
-	const baseFilterd = filter(proxiedHast, (node) => {
+	const baseFilterd = filter(body, (node) => {
 		if (!metadataFilter(node as Hast)) {
 			return false;
 		}
@@ -175,36 +166,41 @@ export const readabilityExtractHast = (hast: Hast): Hast => {
 		return { type: "root", children: [] };
 	}
 
-	const baseText = hastToString(baseFilterd);
+	const baseFilterdText = hastToString(baseFilterd);
+	let [baseTree, baseText] =
+		baseFilterdText.length > bodyText.length / 3 || baseFilterdText.length > 5000
+			? ([baseFilterd as Hast, baseFilterdText] as const)
+			: ([body as Hast, bodyText] as const);
+
 	let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang as keyof typeof BASE_MINIMAL_LENGTH] : 500;
 	if (baseText.length < minimalLength) {
 		minimalLength = Math.max(0, baseText.length - 200);
 	}
 
-	let bodyTree: Hast = baseFilterd;
 	for (const selector of BODY_SELECTORS) {
-		const body = { type: "root" as const, children: selectAll(selector, baseFilterd) };
-		const bodyText = hastToString(body);
+		const content = { type: "root" as const, children: selectAll(selector, baseFilterd) };
+		const contentText = hastToString(content);
 
-		if (bodyText.length < 25) {
+		if (contentText.length < 25) {
 			continue;
 		}
 
-		const links = selectAll("a", body);
+		const links = selectAll("a", content);
 		const linkText = links.map((link) => hastToString(link)).join("");
 
 		const linkDensity = linkText.length / bodyText.length;
 		if (linkDensity > 0.4) {
 			continue;
 		}
 
-		if (bodyText.length > minimalLength) {
-			bodyTree = body;
+		if (contentText.length > minimalLength) {
+			baseTree = content;
+			baseText = contentText;
 			break;
 		}
 	}
 
-	const finalTree = filter(bodyTree, (node) => {
+	const finalFilteredTree = filter(baseTree, (node) => {
 		if (!removeEmptyFilter(node as Hast)) {
 			return false;
 		}
@@ -215,5 +211,6 @@ export const readabilityExtractHast = (hast: Hast): Hast => {
 		return true;
 	}) as Hast;
 
+	const finalTree = hastToString(finalFilteredTree).length > baseText.length / 3 ? finalFilteredTree : baseTree;
 	return finalTree;
 };

packages/webforai/src/html-to-markdown.test.ts

@@ -123,17 +123,17 @@ describe("htmlToMarkdown", () => {
 describe("htmlToMarkdown E2E", () => {
 	it("should convert npm README to Markdown ", async () => {
 		const html = await loadHtml("https://www.npmjs.com/package/webforai");
-		const markdown = htmlToMarkdown(html, { baseUrl: "https://www.npmjs.com/package/webforai" });
+		const markdown = htmlToMarkdown(html);
 
 		// @ts-ignore
 		const original = await import("../README.md?raw");
 		const d = distance(markdown, original.default);
-		expect(d).lte(400); // I'd like to optimise more!
+		expect(d).lte(200); // I'd like to optimise more!
 	});
 
 	it("should convert GitHub README to Markdown ", async () => {
 		const html = await loadHtml("https://github.com/inaridiy/webforai");
-		const markdown = htmlToMarkdown(html, { baseUrl: "https://www.npmjs.com/package/webforai" });
+		const markdown = htmlToMarkdown(html);
 
 		// @ts-ignore
 		const original = await import("../../../README.md?raw");