release: v5.7.0 - Support TOON

snowfluke · snowfluke · commit 4917c911e99d · 2025-11-11T16:28:35.000+07:00
diff --git a/README.md b/README.md
@@ -7,6 +7,7 @@ There are two class of `PdfReader` (uses mupdfjs) and `PdfReaderLegacy` uses (pd
 ## Features
 
 - **Text Extraction:** Retrieve all text content from a PDF.
+- **LLM-Friendly Text Extraction:** Retrieve all text content while maintaining bbox and encode in Token Object Notation (TOON).
 - **Coordinate Data:** Get precise bounding box and dimension information for each text element.
 - **Line Grouping:** Merge individual text tokens into coherent lines.
 - **Scanned PDF Detection:** Determine if a PDF/individual page appears to be scanned or digitally generated.
@@ -16,23 +17,24 @@ There are two class of `PdfReader` (uses mupdfjs) and `PdfReaderLegacy` uses (pd
 
 ## Differences
 
-| Indicator                  | PdfReader | PdfReaderLegacy |
-| -------------------------- | --------- | --------------- |
-| Library                    | mupdfjs   | pdfjs-dist      |
-| Pages index start          | 0         | 1               |
-| open()                     | ✅        | ✅              |
-| getTexts()                 | ✅        | ✅              |
-| getTextsScanned()          | ✅        | ✅              |
-| isScanned()                | ✅        | ✅              |
-| isPageScanned()            | ✅        | ✅              |
-| getLinesFromTexts()        | ✅        | ✅              |
-| getCompactLinesFromTexts() | ✅        | ✅              |
-| destroy()                  | ✅        | ✅              |
-| destroyPage()              | ✅        | ❌              |
-| renderAll()                | ✅        | ✅              |
-| saveCanvasToPng()          | ✅        | ✅              |
-| dumpCanvasMap()            | ✅        | ✅              |
-| Resize viewport/Custom DPI | ✅        | ✅              |
+| Indicator                           | PdfReader | PdfReaderLegacy |
+| ----------------------------------- | --------- | --------------- |
+| Library                             | mupdfjs   | pdfjs-dist      |
+| Pages index start                   | 0         | 1               |
+| open()                              | ✅        | ✅              |
+| getTexts()                          | ✅        | ✅              |
+| getTextsScanned()                   | ✅        | ✅              |
+| isScanned()                         | ✅        | ✅              |
+| isPageScanned()                     | ✅        | ✅              |
+| getLinesFromTexts()                 | ✅        | ✅              |
+| getCompactLinesFromTexts()          | ✅        | ✅              |
+| destroy()                           | ✅        | ✅              |
+| destroyPage()                       | ✅        | ❌              |
+| renderAll()                         | ✅        | ✅              |
+| saveCanvasToPng()                   | ✅        | ✅              |
+| dumpCanvasMap()                     | ✅        | ✅              |
+| Resize viewport/Custom DPI          | ✅        | ✅              |
+| pdfReader.getLinesFromTextsInToon() | ✅        | ✅              |
 
 ## Benchmark
 
@@ -204,6 +206,7 @@ Configuration options for `PdfReader`, allowing customization of PDF text extrac
 | `mergeCloseTextNeighbor`     | `boolean` | `true`        | Merges text elements that are close to each other into a single entity.     |
 | `simpleSortAlgorithm`        | `boolean` | `false`       | Uses a simplified sorting algorithm for text positioning.                   |
 | `scale`                      | `number`  | `1`           | The pdf document scale                                                      |
+| `enableToon`                 | `boolean` | `false`       | To enable pdf words extraction in TOON format                               |
 
 ### Usage Example:
 
diff --git a/bun.lock b/bun.lock
@@ -5,6 +5,7 @@
       "name": "ppu-pdf",
       "dependencies": {
         "@napi-rs/canvas": "^0.1.69",
+        "@toon-format/toon": "^1.0.0",
         "mupdf": "^1.26.2",
         "pdfjs-dist": "4.9.155",
       },
@@ -140,6 +141,8 @@
 
     "@techstark/opencv-js": ["@techstark/opencv-js@4.10.0-release.1", "", {}, "sha512-S4XELidRiQeA0q1s9VQLo540wCxUo24r1O4C+LqZ6llX+sPCXvZCPv3Ice8dEIr0uavyZ8YZeKXSBdDgMXSXjw=="],
 
+    "@toon-format/toon": ["@toon-format/toon@1.0.0", "", {}, "sha512-2gIk8LaqrzpurNDaDWZ72kucAGcBbxJxUnp+4ZP+Pny/QVNdMVf97yzSDTI3ed2q8ypjj8T271P6iE3bRmQBNw=="],
+
     "@types/bun": ["@types/bun@1.3.1", "", { "dependencies": { "bun-types": "1.3.1" } }, "sha512-4jNMk2/K9YJtfqwoAa28c8wK+T7nvJFOjxI4h/7sORWcypRNxBpr+TPNaCfVWq70tLCJsqoFwcf0oI0JU/fvMQ=="],
 
     "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
diff --git a/examples/pdf-digital.example.ts b/examples/pdf-digital.example.ts
@@ -2,7 +2,7 @@
 
 import { PdfReader } from "../src";
 
-const pdfReader = new PdfReader({ verbose: false });
+const pdfReader = new PdfReader({ verbose: false, enableToon: true });
 const file = Bun.file("./assets/opposite-expectation.pdf");
 
 const buffer = await file.arrayBuffer();
@@ -16,5 +16,8 @@ pdfReader.destroy(pdf);
 const lines = pdfReader.getLinesFromTexts(texts);
 console.log("lines: ", lines.get(0));
 
+const linesInToon = pdfReader.getLinesFromTextsInToon(texts);
+console.log("lines in toon: ", linesInToon);
+
 const isScanned = pdfReader.isScanned(texts);
 console.log("is pdf scanned: ", isScanned);
diff --git a/jsr.json b/jsr.json
@@ -1,6 +1,6 @@
 {
   "name": "@snowfluke/ppu-pdf",
-  "version": "5.6.0",
+  "version": "5.7.0",
   "license": "MIT",
   "exports": "./src/index.ts",
   "publish": {
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ppu-pdf",
-  "version": "5.6.0",
+  "version": "5.7.0",
   "description": "Easily extract text from digital PDF files with coordinate and font size included, and optionally group text by lines or render scanned pdf to canvas/png.",
   "keywords": [
     "pdf-reader",
@@ -53,6 +53,7 @@
   },
   "dependencies": {
     "@napi-rs/canvas": "^0.1.69",
+    "@toon-format/toon": "^1.0.0",
     "mupdf": "^1.26.2",
     "pdfjs-dist": "4.9.155"
   }
diff --git a/src/pdf-reader-common.ts b/src/pdf-reader-common.ts
@@ -2,16 +2,19 @@ import { createCanvas, type Canvas } from "@napi-rs/canvas";
 import { createWriteStream, existsSync, mkdirSync } from "fs";
 import { join } from "path";
 
+import { encode } from "@toon-format/toon";
 import { CONSTANT } from "./pdf.constant";
 import {
   type CompactPageLines,
   type CompactPdfLine,
   type CompactPdfWord,
   type PageLines,
   type PageTexts,
+  type PageToonLines,
   type PdfCompactLineAlgorithm,
   type PdfLine,
   type PdfScannedThreshold,
+  type PdfToonLine,
   type PdfWord,
 } from "./pdf.interface";
 
@@ -389,4 +392,40 @@ export class PdfReaderCommon {
 
     return isWordsBelowThreshold || isTextLengthBelowThreshold;
   }
+
+  protected getToonWords(pdfWords: PdfWord[], enableToon: boolean): string {
+    if (!enableToon) return "";
+
+    const simplifyWords = pdfWords.map((word) => ({
+      text: word.text,
+      bbox: [word.bbox.x0, word.bbox.y0, word.bbox.x1, word.bbox.y1],
+    }));
+
+    return encode(simplifyWords);
+  }
+
+  protected getLinesFromTextsInToonCommon(
+    pageTexts: PageTexts,
+    startIndex = 0
+  ): PageToonLines {
+    let pageLines: PageToonLines = "";
+    const numOfPages = pageTexts.size;
+
+    for (let i = startIndex; i < numOfPages + startIndex; i++) {
+      const pdfText = pageTexts.get(i);
+      let lines: PdfToonLine = {};
+      if (pdfText) {
+        lines = this.getLines(pdfText.words).reduce((acc, word, index) => {
+          acc[`${index}`] = word.words.map((el) => ({
+            text: el.text,
+            bbox: [el.bbox.x0, el.bbox.y0, el.bbox.x0, el.bbox.y1],
+          }));
+          return acc;
+        }, {} as PdfToonLine);
+      }
+      pageLines += `# Page ${i} lines:\n ${encode(lines)}\n`;
+    }
+
+    return pageLines;
+  }
 }
diff --git a/src/pdf-reader-legacy.ts b/src/pdf-reader-legacy.ts
@@ -23,6 +23,7 @@ import {
   type CompactPageLines,
   type PageLines,
   type PageTexts,
+  type PageToonLines,
   type PdfCompactLineAlgorithm,
   type PdfReaderOptions,
   type PdfScannedThreshold,
@@ -203,6 +204,8 @@ export class PdfReaderLegacy extends PdfReaderCommon {
     linesMap.set(pageNum, {
       words: textsFiltered,
       fullText,
+      confidence: 1,
+      toon: this.getToonWords(textsFiltered, this.options.enableToon),
     });
   }
 
@@ -235,6 +238,8 @@ export class PdfReaderLegacy extends PdfReaderCommon {
       linesMap.set(pageNum, {
         words: textsFiltered,
         fullText,
+        confidence: ocrResult.confidence,
+        toon: this.getToonWords(textsFiltered, this.options.enableToon),
       });
     } catch (error) {
       if (this.options.verbose) {
@@ -243,6 +248,8 @@ export class PdfReaderLegacy extends PdfReaderCommon {
       linesMap.set(pageNum, {
         words: [],
         fullText: "",
+        confidence: 0,
+        toon: "",
       });
     }
   }
@@ -467,6 +474,15 @@ export class PdfReaderLegacy extends PdfReaderCommon {
     return this.getLinesFromTextsCommon(pageTexts, this.startIndex);
   }
 
+  /**
+   * Converts extracted text into TOON format string for LLM-friendly input.
+   * @param pageTexts - The extracted text data from a PDF.
+   * @returns A string of TOON format
+   */
+  getLinesFromTextsInToon(pageTexts: PageTexts): PageToonLines {
+    return this.getLinesFromTextsInToonCommon(pageTexts, this.startIndex);
+  }
+
   /**
    * Converts extracted text into compact structured lines using a specified algorithm.
    * @param pageTexts - The extracted text data from a PDF.
diff --git a/src/pdf-reader.ts b/src/pdf-reader.ts
@@ -13,6 +13,7 @@ import {
   type CompactPageLines,
   type PageLines,
   type PageTexts,
+  type PageToonLines,
   type PdfCompactLineAlgorithm,
   type PdfReaderOptions,
   type PdfScannedThreshold,
@@ -207,6 +208,8 @@ export class PdfReader extends PdfReaderCommon {
     linesMap.set(pageNum, {
       words: textsFiltered,
       fullText,
+      confidence: 1,
+      toon: this.getToonWords(textsFiltered, this.options.enableToon),
     });
   }
 
@@ -239,6 +242,8 @@ export class PdfReader extends PdfReaderCommon {
       linesMap.set(pageNum, {
         words: textsFiltered,
         fullText,
+        confidence: ocrResult.confidence,
+        toon: this.getToonWords(textsFiltered, this.options.enableToon),
       });
     } catch (error) {
       if (this.options.verbose) {
@@ -247,6 +252,8 @@ export class PdfReader extends PdfReaderCommon {
       linesMap.set(pageNum, {
         words: [],
         fullText: "",
+        confidence: 0,
+        toon: "",
       });
     }
   }
@@ -446,6 +453,15 @@ export class PdfReader extends PdfReaderCommon {
     return this.getLinesFromTextsCommon(pageTexts, this.startIndex);
   }
 
+  /**
+   * Converts extracted text into TOON format string for LLM-friendly input.
+   * @param pageTexts - The extracted text data from a PDF.
+   * @returns A string of TOON format
+   */
+  getLinesFromTextsInToon(pageTexts: PageTexts): PageToonLines {
+    return this.getLinesFromTextsInToonCommon(pageTexts, this.startIndex);
+  }
+
   /**
    * Converts extracted text into compact structured lines using a specified algorithm.
    * @param pageTexts - The extracted text data from a PDF.
diff --git a/src/pdf.constant.ts b/src/pdf.constant.ts
@@ -50,4 +50,7 @@ export const PDF_READER_DEFAULT_OPTIONS: PdfReaderOptions = {
 
   /** List of fonts to be used in the document. */
   fonts: [],
+
+  /** Whether to turn on/off toon notation format extraction for LLM-friendly text */
+  enableToon: false,
 };
diff --git a/src/pdf.interface.ts b/src/pdf.interface.ts
@@ -80,6 +80,17 @@ export interface PdfWord {
   metadata: PdfMetadata;
 }
 
+/**
+ * Represents a single word extracted from a PDF in TOON Format.
+ */
+export interface PdfToonWord {
+  /** The extracted text. */
+  text: string;
+
+  /** The bounding box of the word [x,y,x1,y1] */
+  bbox: [number, number, number, number];
+}
+
 /**
  * Represents all words extracted from a page.
  */
@@ -89,6 +100,12 @@ export interface PdfTexts {
 
   /** The full text of extracted words */
   fullText: string;
+
+  /** The full confidence of text extraction */
+  confidence: number;
+
+  /** The full text of extracted words in toon notation for LLM-friendly input */
+  toon: string;
 }
 
 /**
@@ -111,6 +128,11 @@ export interface PdfLine {
   words: PdfWord[];
 }
 
+/**
+ * Represents a single line of text in a PDF in TOON format.
+ */
+export type PdfToonLine = Record<string, PdfToonWord[]>;
+
 /**
  * Represents a mapping of page numbers to their corresponding extracted texts.
  */
@@ -121,6 +143,11 @@ export type PageTexts = Map<number, PdfTexts>;
  */
 export type PageLines = Map<number, PdfLine[]>;
 
+/**
+ * Represents a lines in a string formmat of TOON.
+ */
+export type PageToonLines = string;
+
 /**
  * Represents a compact version of a PDF word with only text and bounding box.
  */
@@ -200,6 +227,9 @@ export interface PdfReaderOptions {
 
   /** List of fonts used in the document. */
   fonts: PdfFont[];
+
+  /** Whether to turn on/off toon notation format extraction for LLM-friendly text */
+  enableToon: boolean;
 }
 
 /**

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@snowfluke/ppu-pdf",`
`3`		`- "version": "5.6.0",`
	`3`	`+ "version": "5.7.0",`
`4`	`4`	`"license": "MIT",`
`5`	`5`	`"exports": "./src/index.ts",`
`6`	`6`	`"publish": {`