diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml new file mode 100644 index 00000000..c9899d34 --- /dev/null +++ b/.github/workflows/npm-publish.yml @@ -0,0 +1,179 @@ +# Publish the `fleischwolf` npm package (Node.js / Bun native bindings) for a +# chosen release. The package is a native N-API addon, so it can't be a one-line +# `npm publish`: it's built on a matrix of native runners (one per OS/arch), +# each producing a platform `.node`, then a publish job assembles the main +# package plus per-platform `optionalDependencies` (fleischwolf-) via +# napi-rs and publishes them all. +# +# Trigger: manual only (workflow_dispatch). Pick the release tag to build — the +# workflow checks out that tag and publishes the npm version derived from it, so +# npm releases stay decoupled from the crates.io release on every master push. +# Run it from the Actions tab (or `gh workflow run npm-publish.yml -f tag=v0.7.0`). +# +# Requires one repository secret: NPM_TOKEN — an npm automation token with +# publish rights to `fleischwolf` and the `fleischwolf-*` platform packages. + +name: npm publish + +on: + workflow_dispatch: + inputs: + tag: + description: "Release tag to build and publish (e.g. v0.7.0)." + required: true + version: + description: "Override the npm version (e.g. 0.7.0). Blank = derive from the tag." + required: false + default: "" + +concurrency: + group: npm-publish-${{ inputs.tag }} + cancel-in-progress: false + +defaults: + run: + working-directory: crates/fleischwolf-node + +jobs: + build: + name: build ${{ matrix.target }} + runs-on: ${{ matrix.host }} + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + host: ubuntu-22.04 + # GitHub-hosted ARM64 Linux runner (native — avoids cross-compiling the + # ONNX/ort build). Requires the arm64 runner image to be available. + - target: aarch64-unknown-linux-gnu + host: ubuntu-24.04-arm + - target: x86_64-apple-darwin + host: macos-13 + - target: aarch64-apple-darwin + host: macos-14 + - target: x86_64-pc-windows-msvc + host: windows-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag }} + + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - uses: Swatinem/rust-cache@v2 + with: + # One workspace, key the cache per target so the 5 jobs don't collide. + key: ${{ matrix.target }} + workspaces: "." + + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install napi CLI + run: npm install + + # Native addon + the JS loader / d.ts. `--strip` keeps the (ONNX-linked) + # binary as small as possible; strip isn't available under MSVC, so the + # Windows build omits it. + - name: Build (unix) + if: runner.os != 'Windows' + run: npx napi build --platform --release --strip --target ${{ matrix.target }} --js native.js --dts native.d.ts + + - name: Build (windows) + if: runner.os == 'Windows' + run: npx napi build --platform --release --target ${{ matrix.target }} --js native.js --dts native.d.ts + + - name: Upload prebuilt binary + uses: actions/upload-artifact@v4 + with: + name: bindings-${{ matrix.target }} + path: crates/fleischwolf-node/fleischwolf.*.node + if-no-files-found: error + + # The JS loader + types are platform-agnostic; upload them once (from the + # linux-x64 build) for the publish job to include in the main package. + - name: Upload JS binding + if: matrix.target == 'x86_64-unknown-linux-gnu' + uses: actions/upload-artifact@v4 + with: + name: js-binding + path: | + crates/fleischwolf-node/native.js + crates/fleischwolf-node/native.d.ts + if-no-files-found: error + + publish: + name: publish to npm + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag }} + + - uses: actions/setup-node@v4 + with: + node-version: 20 + registry-url: "https://registry.npmjs.org" + + - name: Install napi CLI + run: npm install + + # Prebuilt binaries → artifacts//fleischwolf..node + - name: Download prebuilt binaries + uses: actions/download-artifact@v4 + with: + pattern: bindings-* + path: crates/fleischwolf-node/artifacts + + # The JS loader + types → the package root. + - name: Download JS binding + uses: actions/download-artifact@v4 + with: + name: js-binding + path: crates/fleischwolf-node + + # Version = the explicit override, else the selected tag with its leading `v`. + - name: Resolve version + id: ver + run: | + v="${{ inputs.version }}" + if [ -z "$v" ]; then v="${{ inputs.tag }}"; v="${v#v}"; fi + echo "version=$v" >> "$GITHUB_OUTPUT" + echo "Publishing fleischwolf@$v from tag ${{ inputs.tag }}" + + # Skip cleanly if this version is already on npm (idempotent re-runs). + - name: Check if already published + id: check + run: | + v="${{ steps.ver.outputs.version }}" + if npm view "fleischwolf@$v" version >/dev/null 2>&1; then + echo "published=true" >> "$GITHUB_OUTPUT" + echo "fleischwolf@$v is already on npm — skipping." + else + echo "published=false" >> "$GITHUB_OUTPUT" + fi + + - name: Set package version + if: steps.check.outputs.published == 'false' + run: npm version "${{ steps.ver.outputs.version }}" --no-git-tag-version --allow-same-version + + # Create the per-platform package dirs and move each prebuilt .node into + # its dir. `npm publish` then runs `prepublishOnly` (napi prepublish), which + # publishes the fleischwolf- packages and wires them into the main + # package's optionalDependencies before the main package is published. + - name: Assemble platform packages + if: steps.check.outputs.published == 'false' + run: | + npx napi create-npm-dir -t . + npx napi artifacts --dir artifacts + + - name: Publish + if: steps.check.outputs.published == 'false' + run: npm publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/Cargo.toml b/Cargo.toml index 379185bc..696fd767 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,8 @@ members = [ "crates/fleischwolf", "crates/fleischwolf-cli", "crates/fleischwolf-pdf", + # Node.js / Bun N-API bindings (published to npm, not crates.io). + "crates/fleischwolf-node", ] [workspace.package] diff --git a/README.md b/README.md index ab65e63b..fa7284c7 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,35 @@ buffered `export_to_markdown_with_images` path. Use The CLI streams Markdown by default (`--no-stream` opts back into buffering; `--to json` and `--images referenced` always buffer). +## Node.js / Bun bindings + +Native TypeScript bindings live in +[`crates/fleischwolf-node`](./crates/fleischwolf-node) (built with +[napi-rs](https://napi.rs)). They ship a real `.node` addon that loads in both +Node.js and Bun (Bun implements N-API — same binary, no rebuild), exposing the +converter with the same knobs as the Rust API: Markdown / docling JSON output, +`strict` mode, image modes, allowed-format restriction, `fetchImages`, sync + +async (`Promise`) calls, and a `streamFileMarkdown` async generator. + +```bash +cd crates/fleischwolf-node && npm install && npm run build +``` + +```ts +import { convertFile, convertFileAsync, streamFileMarkdown } from 'fleischwolf' + +const { content } = convertFile('report.docx') // Markdown +const json = await convertFileAsync('paper.pdf', { to: 'json' }) +for await (const chunk of streamFileMarkdown('paper.pdf')) process.stdout.write(chunk) +``` + +Declarative formats work out of the box. The PDF/image pipeline needs pdfium + +the ONNX models (not bundled), so it throws until you call `installDependencies()` +— which auto-downloads pdfium/OCR and fetches the layout/TableFormer ONNX from a +`modelsUrl` you host. A reusable `Pipeline` keeps those models warm across many +PDFs. See [`crates/fleischwolf-node/README.md`](./crates/fleischwolf-node/README.md) +for the full API and runnable Node + Bun examples. + ## Testing All commands run from the `fleischwolf/` workspace root. @@ -286,7 +315,9 @@ fleischwolf — bigger means Rust wins by more. |---|---|---| | `fleischwolf-core` | `DoclingDocument` model + serializers | `docling-core` | | `fleischwolf` | `DocumentConverter`, source loading, backends | `docling` | +| `fleischwolf-pdf` | PDF/image ML pipeline (pdfium + ONNX layout/table/OCR) | `docling` PDF pipeline | | `fleischwolf-cli` | command-line interface | `docling.cli` | +| `fleischwolf-node` | Node.js / Bun N-API bindings (npm package) | — | ## License diff --git a/crates/fleischwolf-node/.gitignore b/crates/fleischwolf-node/.gitignore new file mode 100644 index 00000000..6578782f --- /dev/null +++ b/crates/fleischwolf-node/.gitignore @@ -0,0 +1,18 @@ +# npm +node_modules/ +package-lock.json + +# Build artifacts generated by `napi build` (regenerate with `npm run build`): +# - the native addon (large; carries the linked ONNX runtime) +# - the platform loader + its types +*.node +native.js +native.d.ts + +# Example scratch output +examples/artifacts/ + +# Cross-platform publish scaffolding, generated in CI by `napi create-npm-dir` +# and `napi artifacts` (per-platform packages + downloaded prebuilt binaries) +npm/ +artifacts/ diff --git a/crates/fleischwolf-node/Cargo.toml b/crates/fleischwolf-node/Cargo.toml new file mode 100644 index 00000000..f8456962 --- /dev/null +++ b/crates/fleischwolf-node/Cargo.toml @@ -0,0 +1,37 @@ +# Node.js / Bun bindings for Fleischwolf, built with napi-rs. +# +# Produces a native N-API addon (`.node`) that loads in both Node.js and Bun +# (Bun implements N-API). The TypeScript surface mirrors the Rust +# `DocumentConverter`: convert a file or in-memory bytes to Markdown or docling +# JSON, with the same knobs (strict Markdown, image modes, allowed formats, +# external image fetching) plus incremental Markdown streaming. +# +# This crate is intentionally kept OUT of the default workspace members' publish +# flow (it ships to npm, not crates.io). It is a `cdylib` — the only artifact is +# the addon, there is no Rust library API here. +[package] +name = "fleischwolf-node" +description = "Node.js / Bun bindings for Fleischwolf (a Rust port of docling)." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +readme = "README.md" +# ort (via fleischwolf-pdf) needs a newer compiler than the std-only crates. +rust-version = "1.82" +# Not a crates.io library — the deliverable is the npm package. +publish = false + +[lib] +# napi addons are C-ABI dynamic libraries loaded by the JS runtime. +crate-type = ["cdylib"] + +[dependencies] +fleischwolf = { path = "../fleischwolf", version = "0.6.1" } +# Node-API bindings. `napi4` enables threadsafe functions, used for streaming. +napi = { version = "2", default-features = false, features = ["napi4"] } +napi-derive = "2" + +[build-dependencies] +napi-build = "2" diff --git a/crates/fleischwolf-node/README.md b/crates/fleischwolf-node/README.md new file mode 100644 index 00000000..a36f8e03 --- /dev/null +++ b/crates/fleischwolf-node/README.md @@ -0,0 +1,242 @@ +# fleischwolf (Node.js / Bun bindings) + +Native [Node.js](https://nodejs.org) / [Bun](https://bun.sh) bindings for +[Fleischwolf](https://github.com/artiz/fleischwolf) — a Rust port of +[docling](https://github.com/docling-project/docling). Convert Markdown, HTML, +DOCX, PPTX, XLSX, EPUB, ODF, LaTeX, email, PDF, images and more into a unified +`DoclingDocument`, and export it as **Markdown** or docling-core **JSON**. + +Built with [napi-rs](https://napi.rs), so it ships a real native addon (`.node`) +that loads in both Node.js and Bun (Bun implements N-API) — the same binary, no +rebuild between runtimes. + +## Install + +Released versions ship **prebuilt** native binaries, so no Rust toolchain is +needed to use the package: + +```bash +npm install fleischwolf # or: bun add fleischwolf +``` + +Prebuilt platforms: Linux x64 / arm64 (glibc), macOS x64 / arm64, Windows x64. +The right binary is pulled in automatically as a platform-specific +`optionalDependency` (`fleischwolf-`). Releases are published to npm by +manually running the `npm publish` workflow +(`.github/workflows/npm-publish.yml`) for a chosen release tag — decoupled from +the crates.io release. + +## Build from source + +This package lives in the Fleischwolf Cargo workspace and can also build the +addon from Rust source — needed for local development or an unsupported +platform. You need a Rust toolchain (1.82+) and Node.js 14+ (or Bun). + +```bash +cd crates/fleischwolf-node +npm install # installs @napi-rs/cli +npm run build # release build → fleischwolf..node + native.js/.d.ts +# npm run build:debug # faster, unoptimized +``` + +> The addon statically links the ONNX runtime used by the PDF/image pipeline, so +> the built `.node` is large. Declarative formats (Markdown, HTML, DOCX, …) don't +> touch it; only PDF/image conversion loads the ML models (downloaded on first +> use, like the CLI). + +## Quick start + +```js +import { convertFile, convert, DocumentConverter } from 'fleischwolf' + +// Convert a file — format detected from the extension. +const { content } = convertFile('report.docx') +console.log(content) // Markdown + +// Convert in-memory bytes (e.g. an upload) — pass the format explicitly. +const md = convert({ name: 'notes', data: Buffer.from('# Hi\n'), format: 'md' }) + +// docling-core JSON instead of Markdown. +const json = convertFile('report.docx', { to: 'json' }) + +// Reuse a converter across many documents. +const converter = new DocumentConverter({ strict: true }) +const a = converter.convert({ name: 'a.md', data: Buffer.from('# A\n') }) +``` + +CommonJS works too: `const { convertFile } = require('fleischwolf')`. + +### Async (off the event loop) + +Conversion is CPU-bound; the `*Async` variants run it on the libuv thread pool +so the event loop stays free. Prefer these for PDF/image and for servers. + +```js +import { convertFileAsync } from 'fleischwolf' + +const res = await convertFileAsync('paper.pdf', { to: 'json' }) +``` + +### Streaming Markdown + +`streamFileMarkdown` yields Markdown chunks in document order as conversion +progresses. For PDF (whose pages convert in parallel) output starts flowing +before the whole document is done; concatenating the chunks reproduces the +buffered `content` byte-for-byte. + +```js +import { streamFileMarkdown } from 'fleischwolf' + +for await (const chunk of streamFileMarkdown('paper.pdf')) { + process.stdout.write(chunk) +} +``` + +### PDF / images: installing the ML models + +Declarative formats (Markdown, HTML, DOCX, XLSX, …) are pure Rust and need +nothing. The **PDF/image** path needs native assets that are *not* bundled in the +addon — pdfium plus the ONNX models (layout, OCR, TableFormer) — the same way +Python docling downloads its models on first use. Converting a PDF/image/METS +input **throws** until they're installed: + +```js +import { installDependencies, checkDependencies, convertFileAsync } from 'fleischwolf' + +await convertFileAsync('paper.pdf') // ❌ throws: "requires the PDF/ML dependencies … call installDependencies()" + +await installDependencies() // provisions everything, then: +await convertFileAsync('paper.pdf') // ✅ works +``` + +What `installDependencies()` fetches, into `~/.cache/fleischwolf` (override with +`dir` or `$FLEISCHWOLF_HOME`), wiring the matching `DOCLING_*` / +`PDFIUM_DYNAMIC_LIB_PATH` env vars in-process: + +| Asset | Source | Required for | +| --- | --- | --- | +| **pdfium** | bblanchon prebuilt (auto, platform-detected) | PDF | +| **OCR** rec model + dictionary | HuggingFace / GitHub (auto) | scanned pages | +| **layout** (`layout_heron.onnx`) | your `modelsUrl` (see below) | PDF **and** image | +| **TableFormer** (`tableformer/*.onnx`) | your `modelsUrl` | tables (else geometric fallback) | + +> **layout + TableFormer have no public prebuilt download.** They're PyTorch→ONNX +> exports (`docling-project/docling-layout-heron`, `docling_ibm_models`). Host the +> exported `.onnx` yourself and point `installDependencies` at the base URL via +> `{ modelsUrl }` or `FLEISCHWOLF_MODELS_URL` — it fetches `layout_heron.onnx` and +> `tableformer/{encoder,decoder,bbox}.onnx` from there. Or export them locally +> (repo `scripts/export_layout.py`, `scripts/export_tableformer.py`) and set +> `DOCLING_LAYOUT_ONNX` / `DOCLING_TABLEFORMER_*` — `installDependencies` detects +> those as already installed. Without a layout source it installs pdfium/OCR and +> throws, naming what's missing. + +```js +await installDependencies({ + modelsUrl: 'https://you.example/fleischwolf-models', // serves layout_heron.onnx, tableformer/*.onnx + onProgress: (m) => console.log(m), +}) + +checkDependencies() // { home, pdfium, layout, ocr, tableformer, ready, missing } +``` + +### Reusing a warm `Pipeline` (many PDFs) + +The one-shot `convertFile` / `convertFileAsync` rebuild the pipeline — reloading +every ONNX model — on each call. To convert many PDFs/images, reuse a `Pipeline` +so the models load **once**: + +```js +import { Pipeline } from 'fleischwolf' + +const pipeline = new Pipeline({ strict: true }) +for (const path of pdfPaths) { + const { content } = pipeline.convertFile(path, { to: 'markdown' }) // warm models +} +``` + +`Pipeline` handles `pdf` and `image` inputs (the ML pipeline) and is synchronous +— reuse one instance behind a job queue. + +### Images + +Pick how pictures render in Markdown with `imageMode`: + +```js +// Inline, self-contained: ![Image](data:image/png;base64,…) +convertFile('slides.pptx', { imageMode: 'embedded' }) + +// Referenced: links + the image bytes to write yourself. +const res = convertFile('slides.pptx', { imageMode: 'referenced', artifactsDir: 'assets' }) +for (const img of res.images) { + await fs.writeFile(img.path, img.data) // e.g. assets/image_000000.png +} +``` + +JSON output always embeds extracted images as data URIs. + +## API + +### Functions + +| Function | Returns | Notes | +| --- | --- | --- | +| `convertFile(path, options?)` | `ConvertResult` | Detects format from the extension. | +| `convert(input, options?)` | `ConvertResult` | In-memory bytes (`{ name, data, format? }`). | +| `convertFileAsync(path, options?)` | `Promise` | Off the event loop. | +| `convertAsync(input, options?)` | `Promise` | Off the event loop. | +| `streamFileMarkdown(path, options?)` | `AsyncGenerator` | Markdown chunks in document order. | +| `supportedFormats()` | `string[]` | Supported input format ids. | +| `formatFromName(name)` | `string \| null` | Detect a format id from a filename/extension. | +| `installDependencies(options?)` | `Promise` | Download/validate the PDF/image models + pdfium. | +| `checkDependencies(options?)` | `DependencyStatus` | Report which PDF/image deps are present. | + +`Pipeline` is the reusable warm PDF/image converter: `new Pipeline(converterOptions)` +then `convertFile` / `convert`. + +`DocumentConverter` is the reusable form: `new DocumentConverter(converterOptions)` +then `convert` / `convertFile` / `convertFileAsync` / `convertAsync` / +`convertFileStreaming`. Converter config (`strict`, `fetchImages`, +`allowedFormats`) is set once on the constructor; output options (`to`, +`imageMode`, `artifactsDir`) are per call. + +### Options + +- `to`: `"markdown"` (default) or `"json"`. +- `imageMode`: `"placeholder"` (default), `"embedded"`, or `"referenced"`. +- `artifactsDir`: directory name used in `referenced` links (default `"artifacts"`). +- `strict`: cleaner, more conformant Markdown instead of docling's byte-for-byte + legacy output (Markdown only). +- `fetchImages`: for HTML/EPUB, resolve and embed external ``. Off by + default; fetches http(s) URLs over the network — enable only for trusted input. +- `allowedFormats`: restrict the converter to these format ids/extensions. + +### `ConvertResult` + +```ts +interface ConvertResult { + content: string // Markdown or JSON, per `to` + format: string // detected input format id + status: string // "success" | "partial_success" | "failure" + inputName: string + images: { path: string; data: Buffer }[] // for the `referenced` image mode +} +``` + +Full TypeScript types are generated into `index.d.ts` / `native.d.ts`. + +## Examples + +- [`examples/node-basic.mjs`](examples/node-basic.mjs) — Node.js (ESM): file, bytes, JSON, reuse. +- [`examples/bun-basic.ts`](examples/bun-basic.ts) — Bun + TypeScript, with async and streaming. +- [`examples/pdf-pipeline.mjs`](examples/pdf-pipeline.mjs) — `installDependencies` + warm `Pipeline` for PDFs. + +```bash +npm run build # once +node examples/node-basic.mjs +bun run examples/bun-basic.ts +node test/smoke.mjs # or: bun test/smoke.mjs +``` + +## License + +MIT, same as the rest of Fleischwolf. diff --git a/crates/fleischwolf-node/build.rs b/crates/fleischwolf-node/build.rs new file mode 100644 index 00000000..e457f008 --- /dev/null +++ b/crates/fleischwolf-node/build.rs @@ -0,0 +1,5 @@ +// napi-build wires up the N-API symbol resolution and, on Windows, the +// delay-load shim so the addon links against the host `node.exe`/`bun`. +fn main() { + napi_build::setup(); +} diff --git a/crates/fleischwolf-node/deps.js b/crates/fleischwolf-node/deps.js new file mode 100644 index 00000000..ddd81253 --- /dev/null +++ b/crates/fleischwolf-node/deps.js @@ -0,0 +1,297 @@ +// Dependency provisioning for the PDF/image ML pipeline. +// +// The declarative backends (Markdown, HTML, DOCX, XLSX, …) are pure Rust and +// need nothing. The PDF/image path needs native assets that are NOT bundled in +// the addon (they're large and licensed separately), mirroring how Python +// docling downloads its models on first use: +// +// - libpdfium (PDF text extraction + page rasterization) — required for PDF +// - RT-DETR layout model (models/layout_heron.onnx) — required for PDF & image +// - PP-OCR rec + dict (models/ocr_rec.onnx, ppocr_keys_v1.txt) — used for pages with no text layer +// - TableFormer (models/tableformer/{encoder,decoder,bbox}.onnx) — optional; geometric fallback otherwise +// +// pdfium and the OCR assets have public download URLs and are fetched +// automatically. The layout and TableFormer models are exported from PyTorch +// (docling-project/docling-layout-heron and docling_ibm_models) and have no +// public prebuilt `.onnx`, so they are downloaded from a base URL you provide +// via `installDependencies({ modelsUrl })` or the `FLEISCHWOLF_MODELS_URL` +// environment variable (point it at a host serving `layout_heron.onnx` and +// `tableformer/*.onnx`). If you exported them locally, set `DOCLING_LAYOUT_ONNX` +// etc. and they'll be detected as already installed. +// +// Everything is installed under a single home directory (default +// `~/.cache/fleischwolf`, overridable via `FLEISCHWOLF_HOME` or the `dir` +// option), and the corresponding `DOCLING_*` / `PDFIUM_DYNAMIC_LIB_PATH` +// environment variables are set in-process so the native pipeline finds them. + +'use strict' + +const fs = require('fs') +const os = require('os') +const path = require('path') +const http = require('http') +const https = require('https') +const { execFileSync } = require('child_process') + +// Formats whose conversion requires the ML models + native libs above. +const ML_FORMATS = new Set(['pdf', 'image', 'mets_gbs']) + +const PDFIUM_RELEASE = + 'https://github.com/bblanchon/pdfium-binaries/releases/latest/download' +const OCR_REC_URL = + 'https://huggingface.co/SWHL/RapidOCR/resolve/main/PP-OCRv3/ch_PP-OCRv3_rec_infer.onnx' +const OCR_DICT_URL = + 'https://raw.githubusercontent.com/PaddlePaddle/PaddleOCR/main/ppocr/utils/ppocr_keys_v1.txt' + +// pdfium-binaries platform tag + shared-library filename, by (platform, arch). +function pdfiumPlatform() { + const arch = process.arch === 'arm64' ? 'arm64' : process.arch === 'x64' ? 'x64' : process.arch + switch (process.platform) { + case 'linux': + return { tag: `linux-${arch}`, lib: 'libpdfium.so' } + case 'darwin': + return { tag: `mac-${arch}`, lib: 'libpdfium.dylib' } + case 'win32': + return { tag: `win-${arch}`, lib: 'pdfium.dll' } + default: + throw new Error(`unsupported platform for pdfium: ${process.platform}/${process.arch}`) + } +} + +/** Resolve the install home directory (absolute). */ +function homeDir(dir) { + if (dir) return path.resolve(dir) + if (process.env.FLEISCHWOLF_HOME) return path.resolve(process.env.FLEISCHWOLF_HOME) + return path.join(os.homedir(), '.cache', 'fleischwolf') +} + +/** + * The resolved on-disk location of each dependency: an existing `DOCLING_*` / + * `PDFIUM_DYNAMIC_LIB_PATH` environment variable wins (so a local Python export + * is honored), else the path under the install home directory. + */ +function resolvePaths(dir) { + const home = homeDir(dir) + const models = path.join(home, 'models') + const { lib } = pdfiumPlatform() + + const pdfiumLibDir = process.env.PDFIUM_DYNAMIC_LIB_PATH || path.join(home, 'pdfium', 'lib') + return { + home, + models, + pdfiumLibDir, + pdfiumLib: path.join(pdfiumLibDir, lib), + layout: process.env.DOCLING_LAYOUT_ONNX || path.join(models, 'layout_heron.onnx'), + ocrRec: process.env.DOCLING_OCR_REC_ONNX || path.join(models, 'ocr_rec.onnx'), + ocrDict: process.env.DOCLING_OCR_DICT || path.join(models, 'ppocr_keys_v1.txt'), + tfEncoder: + process.env.DOCLING_TABLEFORMER_ENCODER || path.join(models, 'tableformer', 'encoder.onnx'), + tfDecoder: + process.env.DOCLING_TABLEFORMER_DECODER || path.join(models, 'tableformer', 'decoder.onnx'), + tfBbox: process.env.DOCLING_TABLEFORMER_BBOX || path.join(models, 'tableformer', 'bbox.onnx'), + } +} + +/** + * Report which dependencies are present on disk, without downloading anything. + * `ready` is true when the minimum for PDF (pdfium + layout) is present. + */ +function checkDependencies(options = {}) { + const p = resolvePaths(options.dir) + const has = (f) => fs.existsSync(f) + const status = { + home: p.home, + pdfium: has(p.pdfiumLib), + layout: has(p.layout), + ocr: has(p.ocrRec) && has(p.ocrDict), + tableformer: has(p.tfEncoder) && has(p.tfDecoder) && has(p.tfBbox), + } + status.ready = status.pdfium && status.layout + status.missing = [ + !status.pdfium && 'pdfium', + !status.layout && 'layout_heron.onnx', + ].filter(Boolean) + return status +} + +/** Point the current process at installed assets (so the native pipeline finds them). */ +function exportEnv(p) { + if (fs.existsSync(p.pdfiumLib)) process.env.PDFIUM_DYNAMIC_LIB_PATH = p.pdfiumLibDir + if (fs.existsSync(p.layout)) process.env.DOCLING_LAYOUT_ONNX = p.layout + if (fs.existsSync(p.ocrRec)) process.env.DOCLING_OCR_REC_ONNX = p.ocrRec + if (fs.existsSync(p.ocrDict)) process.env.DOCLING_OCR_DICT = p.ocrDict + if (fs.existsSync(p.tfEncoder)) process.env.DOCLING_TABLEFORMER_ENCODER = p.tfEncoder + if (fs.existsSync(p.tfDecoder)) process.env.DOCLING_TABLEFORMER_DECODER = p.tfDecoder + if (fs.existsSync(p.tfBbox)) process.env.DOCLING_TABLEFORMER_BBOX = p.tfBbox +} + +/** + * Throw a clear, actionable error if `format` needs the ML pipeline but its + * dependencies aren't installed. Called before ML conversions. + */ +function assertMlReady(format, dir) { + if (!ML_FORMATS.has(format)) return + const status = checkDependencies({ dir }) + // Image needs layout (+OCR), but not pdfium; PDF/METS need both. + const needPdfium = format !== 'image' + const missing = [!status.layout && 'layout_heron.onnx', needPdfium && !status.pdfium && 'pdfium'].filter( + Boolean, + ) + if (missing.length === 0) return + throw new Error( + `Converting '${format}' requires the PDF/ML dependencies, which are not installed: ` + + `${missing.join(', ')}. Call \`await installDependencies()\` before converting ` + + `(pass { modelsUrl } or set FLEISCHWOLF_MODELS_URL so the layout/TableFormer ONNX can be ` + + `fetched; or set DOCLING_LAYOUT_ONNX / PDFIUM_DYNAMIC_LIB_PATH to local files). ` + + `Declarative formats (md, html, docx, xlsx, …) need none of this.`, + ) +} + +// --- downloading ----------------------------------------------------------- + +function download(url, dest, onProgress) { + return new Promise((resolve, reject) => { + const tmp = `${dest}.download` + const client = url.startsWith('http://') ? http : https + const req = client.get(url, { headers: { 'User-Agent': 'fleischwolf-node' } }, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + res.resume() + return download(res.headers.location, dest, onProgress).then(resolve, reject) + } + if (res.statusCode !== 200) { + res.resume() + return reject(new Error(`GET ${url} → HTTP ${res.statusCode}`)) + } + fs.mkdirSync(path.dirname(dest), { recursive: true }) + const out = fs.createWriteStream(tmp) + res.pipe(out) + out.on('finish', () => out.close(() => { + fs.renameSync(tmp, dest) + resolve(dest) + })) + out.on('error', reject) + }) + req.on('error', reject) + }) +} + +async function ensureFile(dest, url, force, onProgress, label) { + if (!force && fs.existsSync(dest)) return false + onProgress?.(`downloading ${label}`) + await download(url, dest, onProgress) + return true +} + +async function installPdfium(p, force, onProgress) { + if (!force && fs.existsSync(p.pdfiumLib)) return false + if (process.env.PDFIUM_DYNAMIC_LIB_PATH) { + // The user pointed us at a pdfium directory that doesn't contain the lib. + throw new Error( + `PDFIUM_DYNAMIC_LIB_PATH is set to '${p.pdfiumLibDir}' but no pdfium library was found there.`, + ) + } + const { tag } = pdfiumPlatform() + const url = `${PDFIUM_RELEASE}/pdfium-${tag}.tgz` + const home = p.home + const pdfiumRoot = path.join(home, 'pdfium') + fs.mkdirSync(pdfiumRoot, { recursive: true }) + const tgz = path.join(pdfiumRoot, 'pdfium.tgz') + onProgress?.(`downloading pdfium (${tag})`) + await download(url, tgz) + onProgress?.('extracting pdfium') + // pdfium-binaries ships a .tgz; use the system `tar` (present on Linux, macOS, + // and Windows 10+). The archive lays out lib/ which matches pdfiumLibDir. + execFileSync('tar', ['-xzf', tgz, '-C', pdfiumRoot]) + fs.rmSync(tgz, { force: true }) + if (!fs.existsSync(p.pdfiumLib)) { + throw new Error(`pdfium extracted but ${p.pdfiumLib} is missing (unexpected archive layout)`) + } + return true +} + +/** + * Download and install everything the PDF/image pipeline needs, then point the + * process at it. Idempotent: skips assets already present (pass `{ force: true }` + * to re-download). Returns a status report. + * + * @param {object} [options] + * @param {string} [options.dir] install home (default ~/.cache/fleischwolf or $FLEISCHWOLF_HOME) + * @param {string} [options.modelsUrl] base URL serving layout_heron.onnx + tableformer/*.onnx + * @param {boolean} [options.ocr=true] also fetch the OCR model + dictionary + * @param {boolean} [options.tableformer=true] also fetch TableFormer from modelsUrl (if provided) + * @param {boolean} [options.force=false] re-download assets that already exist + * @param {(msg: string) => void} [options.onProgress] + */ +async function installDependencies(options = {}) { + const p = resolvePaths(options.dir) + const onProgress = options.onProgress + const installed = [] + const missing = [] + fs.mkdirSync(p.models, { recursive: true }) + + // 1. pdfium (required for PDF). + if (await installPdfium(p, options.force, onProgress)) installed.push('pdfium') + + // 2. OCR recognition model + dictionary (for pages without a text layer). + if (options.ocr !== false) { + if (await ensureFile(p.ocrRec, OCR_REC_URL, options.force, onProgress, 'OCR model')) + installed.push('ocr_rec.onnx') + if (await ensureFile(p.ocrDict, OCR_DICT_URL, options.force, onProgress, 'OCR dictionary')) + installed.push('ppocr_keys_v1.txt') + } + + // 3. Layout (required) + TableFormer (optional) — from the configured base URL. + const base = (options.modelsUrl || process.env.FLEISCHWOLF_MODELS_URL || '').replace(/\/$/, '') + if (!fs.existsSync(p.layout)) { + if (base) { + if ( + await ensureFile(p.layout, `${base}/layout_heron.onnx`, options.force, onProgress, 'layout model') + ) + installed.push('layout_heron.onnx') + } else { + missing.push('layout_heron.onnx') + } + } + if (options.tableformer !== false && base) { + for (const [file, dest] of [ + ['tableformer/encoder.onnx', p.tfEncoder], + ['tableformer/decoder.onnx', p.tfDecoder], + ['tableformer/bbox.onnx', p.tfBbox], + ]) { + try { + if (await ensureFile(dest, `${base}/${file}`, options.force, onProgress, file)) + installed.push(file) + } catch (e) { + // TableFormer is optional (geometric fallback); note but don't fail. + onProgress?.(`skipped ${file}: ${e.message}`) + } + } + } + + exportEnv(p) + const status = checkDependencies(options) + + if (!status.ready) { + const hint = base + ? `layout_heron.onnx could not be fetched from ${base}.` + : `no models URL configured — pass { modelsUrl } to installDependencies() or set ` + + `FLEISCHWOLF_MODELS_URL to a host serving layout_heron.onnx (and tableformer/*.onnx). ` + + `The layout model is a PyTorch→ONNX export (docling-project/docling-layout-heron) with no ` + + `public prebuilt download; export it with the repo's scripts/export_layout.py and host it, ` + + `or set DOCLING_LAYOUT_ONNX to the local file.` + throw new Error( + `installDependencies: PDF conversion is not ready. Missing: ${status.missing.join(', ')}. ${hint}`, + ) + } + + return { ...status, installed, missing } +} + +module.exports = { + ML_FORMATS, + installDependencies, + checkDependencies, + assertMlReady, + resolvePaths, + exportEnv, +} diff --git a/crates/fleischwolf-node/examples/bun-basic.ts b/crates/fleischwolf-node/examples/bun-basic.ts new file mode 100644 index 00000000..0ee82fc4 --- /dev/null +++ b/crates/fleischwolf-node/examples/bun-basic.ts @@ -0,0 +1,44 @@ +// Fleischwolf from Bun, in TypeScript. Run with: +// +// bun run examples/bun-basic.ts +// +// Bun implements N-API, so the exact same native addon loads — no rebuild. This +// example leans on the TypeScript types and shows async conversion and the +// streaming async generator. It also runs unchanged under `tsx`/`ts-node` on Node. + +import { fileURLToPath } from 'node:url' +import { dirname, join } from 'node:path' + +import { + convertFileAsync, + streamFileMarkdown, + DocumentConverter, + type ConvertResult, +} from 'fleischwolf' + +const here = dirname(fileURLToPath(import.meta.url)) +const html = join(here, 'inputs', 'sample.html') + +// 1. Async conversion — the CPU-bound work runs off the event loop, so this +// scales to PDF/image without blocking. Fully typed: `res` is ConvertResult. +const res: ConvertResult = await convertFileAsync(html, { to: 'markdown' }) +console.log(`converted "${res.inputName}" (${res.format}) → ${res.status}`) +console.log(res.content) + +// 2. Streaming Markdown: chunks arrive in document order as conversion +// progresses. For PDF, pages stream out as each finishes — output starts +// before the whole document is done. Concatenating the chunks equals the +// buffered `content`. +console.log('--- streamed ---') +let streamed = '' +for await (const chunk of streamFileMarkdown(html)) { + process.stdout.write(chunk) + streamed += chunk +} +console.log(`\n(streamed ${streamed.length} bytes)`) + +// 3. Embedded images: pictures become inline base64 data URIs, so the Markdown +// is self-contained. (This HTML has none; the option is a no-op here, but +// the same call handles DOCX/PPTX/PDF figures.) +const embedded = new DocumentConverter().convertFile(html, { imageMode: 'embedded' }) +console.log('embedded-image Markdown length:', embedded.content.length) diff --git a/crates/fleischwolf-node/examples/inputs/sample.html b/crates/fleischwolf-node/examples/inputs/sample.html new file mode 100644 index 00000000..eba1e386 --- /dev/null +++ b/crates/fleischwolf-node/examples/inputs/sample.html @@ -0,0 +1,24 @@ + + + + Fleischwolf sample + + +

Quarterly report

+

Revenue grew 18% year over year.

+

Highlights

+
    +
  • Launched the new pipeline
    • +
  • Signed three enterprise customers
    • +
+ + + + + + + + +
RegionGrowth
EMEA22%
APAC15%
+ + diff --git a/crates/fleischwolf-node/examples/node-basic.mjs b/crates/fleischwolf-node/examples/node-basic.mjs new file mode 100644 index 00000000..800b6971 --- /dev/null +++ b/crates/fleischwolf-node/examples/node-basic.mjs @@ -0,0 +1,51 @@ +// Fleischwolf from Node.js (ESM). Run with: +// +// node examples/node-basic.mjs +// +// Shows the four common paths: convert a file, convert in-memory bytes, emit +// docling-core JSON, and reuse a converter. See bun-basic.ts for TypeScript and +// streaming. + +import { fileURLToPath } from 'node:url' +import { dirname, join } from 'node:path' + +import { + convertFile, + convert, + DocumentConverter, + supportedFormats, +} from 'fleischwolf' + +const here = dirname(fileURLToPath(import.meta.url)) + +// 1. Convert a file on disk — format detected from the extension. +const html = join(here, 'inputs', 'sample.html') +const doc = convertFile(html) +console.log('--- Markdown from sample.html ---') +console.log(doc.content) + +// 2. Convert in-memory bytes (e.g. an upload) — pass the format explicitly. +const md = '# Notes\n\nInline math $E = mc^2$ and a [link](https://example.com).\n' +const fromBytes = convert({ name: 'notes', data: Buffer.from(md), format: 'md' }) +console.log('--- Round-tripped Markdown ---') +console.log(fromBytes.content) + +// 3. Emit docling-core's native JSON wire format instead of Markdown. +const asJson = convertFile(html, { to: 'json' }) +const model = JSON.parse(asJson.content) +console.log('--- docling JSON ---') +console.log('schema:', model.schema_name, model.version) +console.log('text nodes:', model.texts.length, '| tables:', model.tables.length) + +// 4. Reuse a converter across many documents (config parsed once). `strict` +// yields cleaner, more conformant Markdown. +const converter = new DocumentConverter({ strict: true }) +for (const [name, body] of [ + ['a.md', '# First\n'], + ['b.md', '# Second\n'], +]) { + const r = converter.convert({ name, data: Buffer.from(body) }) + console.log(`converted ${name}: ${r.content.trim()}`) +} + +console.log('\nsupported formats:', supportedFormats().join(', ')) diff --git a/crates/fleischwolf-node/examples/pdf-pipeline.mjs b/crates/fleischwolf-node/examples/pdf-pipeline.mjs new file mode 100644 index 00000000..ae0dbb7f --- /dev/null +++ b/crates/fleischwolf-node/examples/pdf-pipeline.mjs @@ -0,0 +1,42 @@ +// Converting PDFs/images: installing the ML dependencies and reusing a warm +// Pipeline. Run with: +// +// node examples/pdf-pipeline.mjs path/to/document.pdf +// +// The PDF/image path needs native assets that aren't bundled in the addon +// (pdfium + the layout/OCR/TableFormer ONNX models). `installDependencies()` +// provisions them; without it, converting a PDF throws a clear error. + +import { installDependencies, checkDependencies, convertFileAsync, Pipeline } from 'fleischwolf' + +const file = process.argv[2] ?? 'document.pdf' + +// 1. Without the models, PDF conversion throws — the guard points you here. +console.log('deps before:', checkDependencies()) +try { + await convertFileAsync(file) +} catch (err) { + console.log('\nexpected (models not installed yet):\n ', err.message.split('.')[0], '…\n') +} + +// 2. Install them. pdfium + OCR download automatically; the layout + TableFormer +// ONNX come from a base URL you host (or set FLEISCHWOLF_MODELS_URL). If you +// exported the models locally, set DOCLING_LAYOUT_ONNX etc. instead and this +// call just validates + wires them up. +await installDependencies({ + modelsUrl: process.env.FLEISCHWOLF_MODELS_URL, // e.g. https://you.example/fleischwolf-models + onProgress: (m) => console.log(' ·', m), +}) +console.log('deps after:', checkDependencies(), '\n') + +// 3. Convert. For a single file, convertFileAsync is fine (runs off the event loop): +const res = await convertFileAsync(file, { to: 'markdown' }) +console.log(res.content.slice(0, 500)) + +// 4. For MANY PDFs, reuse a warm Pipeline so the models load once instead of per call: +const pipeline = new Pipeline({ strict: true }) +for (const path of process.argv.slice(2)) { + const r = pipeline.convertFile(path, { to: 'json' }) + const doc = JSON.parse(r.content) + console.log(`${path}: ${doc.texts.length} text nodes, ${doc.tables.length} tables`) +} diff --git a/crates/fleischwolf-node/index.d.ts b/crates/fleischwolf-node/index.d.ts new file mode 100644 index 00000000..7913ef2c --- /dev/null +++ b/crates/fleischwolf-node/index.d.ts @@ -0,0 +1,122 @@ +// Public type surface for the `fleischwolf` npm package. Re-exports the native +// binding's option/result types and unguarded functions, and declares the +// JS-wrapped classes, the dependency API, and the streaming helper. + +import type { + ConverterOptions, + OutputOptions, + ConvertOptions, + ConvertInput, + ConvertResult, +} from './native' + +export type { ConverterOptions, OutputOptions, ConvertOptions, ConvertInput, ConvertResult } + +// Format helpers pass straight through from the native binding. +export { supportedFormats, formatFromName } from './native' + +/** Callback form used by the native streaming API (prefer {@link streamFileMarkdown}). */ +export type StreamCallback = (err: Error | null, chunk: string | undefined | null) => void + +/** Convert a file on disk (format detected from the extension). Throws for PDF/image/METS if deps aren't installed. */ +export declare function convertFile(path: string, options?: ConvertOptions | null): ConvertResult +/** Convert in-memory bytes. Throws for PDF/image/METS if deps aren't installed. */ +export declare function convert(input: ConvertInput, options?: ConvertOptions | null): ConvertResult +/** Async (Promise) file conversion, off the event loop. Rejects for PDF/image/METS if deps aren't installed. */ +export declare function convertFileAsync(path: string, options?: ConvertOptions | null): Promise +/** Async (Promise) bytes conversion, off the event loop. */ +export declare function convertAsync(input: ConvertInput, options?: ConvertOptions | null): Promise + +/** A reusable converter holding config (strict / fetchImages / allowedFormats). */ +export declare class DocumentConverter { + constructor(options?: ConverterOptions | null) + convertFile(path: string, options?: OutputOptions | null): ConvertResult + convert(input: ConvertInput, options?: OutputOptions | null): ConvertResult + convertFileAsync(path: string, options?: OutputOptions | null): Promise + convertAsync(input: ConvertInput, options?: OutputOptions | null): Promise + convertFileStreaming(path: string, callback: StreamCallback, options?: OutputOptions | null): void +} + +/** + * A reusable PDF/image pipeline that keeps the ONNX models loaded across calls. + * Use instead of the per-call functions when converting many PDFs/images — the + * one-shot path reloads every model each call. Handles `pdf` and `image` inputs. + */ +export declare class Pipeline { + constructor(options?: ConverterOptions | null) + convertFile(path: string, options?: OutputOptions | null): ConvertResult + convert(input: ConvertInput, options?: OutputOptions | null): ConvertResult +} + +// --- dependency provisioning (PDF/image ML pipeline) ----------------------- + +/** Where installed dependencies live and which are present. */ +export interface DependencyStatus { + /** Install home directory. */ + home: string + /** libpdfium present. */ + pdfium: boolean + /** Layout model (layout_heron.onnx) present. */ + layout: boolean + /** OCR model + dictionary present. */ + ocr: boolean + /** TableFormer encoder/decoder/bbox present. */ + tableformer: boolean + /** True when the minimum for PDF (pdfium + layout) is present. */ + ready: boolean + /** Human-readable list of the missing required assets. */ + missing: string[] +} + +/** Options for {@link installDependencies}. */ +export interface InstallOptions { + /** Install home (default `~/.cache/fleischwolf` or `$FLEISCHWOLF_HOME`). */ + dir?: string + /** Base URL serving `layout_heron.onnx` and `tableformer/*.onnx`. Falls back to `$FLEISCHWOLF_MODELS_URL`. */ + modelsUrl?: string + /** Also fetch the OCR model + dictionary (default true). */ + ocr?: boolean + /** Also fetch TableFormer from `modelsUrl` if provided (default true). */ + tableformer?: boolean + /** Re-download assets that already exist (default false). */ + force?: boolean + /** Progress callback. */ + onProgress?: (message: string) => void +} + +/** + * Download and install everything the PDF/image pipeline needs (pdfium + OCR + * always; layout + TableFormer from `modelsUrl`/`FLEISCHWOLF_MODELS_URL`), then + * point the process at it. Idempotent. Rejects if PDF conversion cannot be made + * ready (e.g. no models URL and no local layout model). + * + * @example + * import { installDependencies, convertFileAsync } from 'fleischwolf' + * await installDependencies({ modelsUrl: 'https://example.com/fleischwolf-models' }) + * const res = await convertFileAsync('paper.pdf', { to: 'markdown' }) + */ +export declare function installDependencies(options?: InstallOptions): Promise + +/** Report which PDF/image dependencies are present, without downloading. */ +export declare function checkDependencies(options?: { dir?: string }): DependencyStatus + +// --- streaming -------------------------------------------------------------- + +/** Options for {@link streamFileMarkdown} (converter config + streamable output). */ +export interface StreamOptions { + strict?: boolean + fetchImages?: boolean + allowedFormats?: string[] + imageMode?: 'placeholder' | 'embedded' + artifactsDir?: string +} + +/** + * Stream a file's Markdown in chunks, in document order, as conversion + * progresses — the win for PDF, whose pages convert in parallel. Concatenating + * the chunks reproduces the buffered `convertFile(path).content` byte-for-byte. + */ +export declare function streamFileMarkdown( + filePath: string, + options?: StreamOptions, +): AsyncGenerator diff --git a/crates/fleischwolf-node/index.js b/crates/fleischwolf-node/index.js new file mode 100644 index 00000000..602609bc --- /dev/null +++ b/crates/fleischwolf-node/index.js @@ -0,0 +1,176 @@ +// Public entry point for the `fleischwolf` npm package. +// +// Wraps the native N-API binding (loaded by `native.js`, which picks the right +// prebuilt `.node` for the host platform) with two things: +// 1. dependency guards — converting a PDF/image/METS input throws a clear +// error unless the ML models + pdfium are installed (see installDependencies); +// 2. a `streamFileMarkdown` async generator over Markdown chunks. +// +// Works in Node.js and Bun (Bun implements N-API). + +'use strict' + +const native = require('./native.js') +const { + installDependencies, + checkDependencies, + assertMlReady, +} = require('./deps.js') + +// Resolve the format id of an input for the ML guard. Uses the native +// extension→format map; falls back to an explicitly-passed format string. +function mlFormatOf(name, format) { + if (format) { + return native.formatFromName(`x.${String(format).replace(/^\./, '')}`) || String(format) + } + return native.formatFromName(name || '') || '' +} + +// --- guarded one-shot functions -------------------------------------------- + +function convertFile(path, options) { + assertMlReady(mlFormatOf(path)) + return native.convertFile(path, options) +} + +function convert(input, options) { + assertMlReady(mlFormatOf(input && input.name, input && input.format)) + return native.convert(input, options) +} + +// async so a guard failure surfaces as a rejected promise, not a sync throw. +async function convertFileAsync(path, options) { + assertMlReady(mlFormatOf(path)) + return native.convertFileAsync(path, options) +} + +async function convertAsync(input, options) { + assertMlReady(mlFormatOf(input && input.name, input && input.format)) + return native.convertAsync(input, options) +} + +// --- guarded classes -------------------------------------------------------- + +class DocumentConverter { + constructor(options) { + this._inner = new native.DocumentConverter(options) + } + + convertFile(path, options) { + assertMlReady(mlFormatOf(path)) + return this._inner.convertFile(path, options) + } + + convert(input, options) { + assertMlReady(mlFormatOf(input && input.name, input && input.format)) + return this._inner.convert(input, options) + } + + async convertFileAsync(path, options) { + assertMlReady(mlFormatOf(path)) + return this._inner.convertFileAsync(path, options) + } + + async convertAsync(input, options) { + assertMlReady(mlFormatOf(input && input.name, input && input.format)) + return this._inner.convertAsync(input, options) + } + + convertFileStreaming(path, callback, options) { + assertMlReady(mlFormatOf(path)) + return this._inner.convertFileStreaming(path, callback, options) + } +} + +// The warm PDF/image pipeline is inherently ML — always guarded. +class Pipeline { + constructor(options) { + this._inner = new native.Pipeline(options) + } + + convertFile(path, options) { + assertMlReady(mlFormatOf(path)) + return this._inner.convertFile(path, options) + } + + convert(input, options) { + assertMlReady(mlFormatOf(input && input.name, input && input.format)) + return this._inner.convert(input, options) + } +} + +// --- streaming -------------------------------------------------------------- + +/** + * Stream a file's Markdown in chunks, in document order, as conversion + * progresses — the win for PDF, whose pages convert in parallel. + * + * Yields each Markdown chunk; concatenating every chunk reproduces the buffered + * `convertFile(path).content` byte-for-byte. + * + * @param {string} filePath + * @param {object} [options] + * @returns {AsyncGenerator} + */ +async function* streamFileMarkdown(filePath, options = {}) { + assertMlReady(mlFormatOf(filePath)) + const { strict, fetchImages, allowedFormats, imageMode, artifactsDir } = options + const converter = new native.DocumentConverter({ strict, fetchImages, allowedFormats }) + + // Bridge the native (err, chunk) callback into an async generator. Chunks are + // delivered on the event loop (via a threadsafe function); a null chunk ends + // the stream, a non-null err ends it with a throw. + const queue = [] + let done = false + let failure = null + let notify = null + const wake = () => { + if (notify) { + const n = notify + notify = null + n() + } + } + + converter.convertFileStreaming( + filePath, + (err, chunk) => { + if (err) { + failure = err + done = true + } else if (chunk === null || chunk === undefined) { + done = true + } else { + queue.push(chunk) + } + wake() + }, + { imageMode, artifactsDir }, + ) + + while (true) { + if (queue.length > 0) { + yield queue.shift() + continue + } + if (failure) throw failure + if (done) return + await new Promise((resolve) => { + notify = resolve + }) + } +} + +// --- exports (explicit, so ESM named imports work in Node and Bun) ---------- + +module.exports.convert = convert +module.exports.convertFile = convertFile +module.exports.convertAsync = convertAsync +module.exports.convertFileAsync = convertFileAsync +module.exports.DocumentConverter = DocumentConverter +module.exports.Pipeline = Pipeline +module.exports.streamFileMarkdown = streamFileMarkdown +module.exports.installDependencies = installDependencies +module.exports.checkDependencies = checkDependencies +module.exports.supportedFormats = native.supportedFormats +module.exports.formatFromName = native.formatFromName diff --git a/crates/fleischwolf-node/package.json b/crates/fleischwolf-node/package.json new file mode 100644 index 00000000..726c6460 --- /dev/null +++ b/crates/fleischwolf-node/package.json @@ -0,0 +1,68 @@ +{ + "name": "fleischwolf", + "version": "0.6.1", + "description": "Node.js / Bun bindings for Fleischwolf — a Rust port of docling. Convert Markdown, HTML, DOCX, PPTX, XLSX, PDF, images and more into a unified DoclingDocument (Markdown or docling-core JSON).", + "keywords": [ + "docling", + "document", + "pdf", + "markdown", + "docx", + "converter", + "napi", + "bun" + ], + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/artiz/fleischwolf.git", + "directory": "crates/fleischwolf-node" + }, + "homepage": "https://github.com/artiz/fleischwolf", + "main": "index.js", + "types": "index.d.ts", + "exports": { + ".": { + "types": "./index.d.ts", + "import": "./index.js", + "require": "./index.js" + } + }, + "napi": { + "name": "fleischwolf", + "triples": { + "defaults": false, + "additional": [ + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc" + ] + } + }, + "engines": { + "node": ">= 14" + }, + "files": [ + "index.js", + "index.d.ts", + "deps.js", + "native.js", + "native.d.ts", + "README.md" + ], + "scripts": { + "build": "napi build --platform --release --strip --js native.js --dts native.d.ts", + "build:debug": "napi build --platform --js native.js --dts native.d.ts", + "artifacts": "napi artifacts", + "prepublishOnly": "napi prepublish -t npm --skip-gh-release", + "version": "napi version", + "test": "node test/smoke.mjs", + "example": "node examples/node-basic.mjs", + "example:bun": "bun run examples/bun-basic.ts" + }, + "devDependencies": { + "@napi-rs/cli": "^2.18.4" + } +} diff --git a/crates/fleischwolf-node/src/lib.rs b/crates/fleischwolf-node/src/lib.rs new file mode 100644 index 00000000..88a2e0b3 --- /dev/null +++ b/crates/fleischwolf-node/src/lib.rs @@ -0,0 +1,751 @@ +//! Node.js / Bun bindings for Fleischwolf, via napi-rs. +//! +//! The surface mirrors the Rust `DocumentConverter`: convert a file (or +//! in-memory bytes) to Markdown or docling-core JSON, with the same options — +//! strict Markdown, picture image modes, allowed-format restriction, external +//! `` fetching — plus incremental Markdown streaming. Everything here is +//! thin glue; the conversion logic lives in the `fleischwolf` crate. +//! +//! Two ways to call it: +//! - the module-level [`convert_file`] / [`convert`] (+ their `*_async` +//! variants), for one-shot use; +//! - the [`DocumentConverter`] class, which holds converter config so it can be +//! reused across many documents. + +use std::cell::RefCell; + +use napi::bindgen_prelude::*; +use napi::threadsafe_function::{ErrorStrategy, ThreadsafeFunction, ThreadsafeFunctionCallMode}; +use napi_derive::napi; + +use fleischwolf::{ + ConversionStatus, DoclingDocument, DocumentConverter as RsConverter, ImageMode, InputFormat, + Pipeline as RsPipeline, SourceDocument, +}; + +// --------------------------------------------------------------------------- +// Options / result shapes exposed to TypeScript. +// --------------------------------------------------------------------------- + +/// Config for a reusable [`DocumentConverter`]. +#[napi(object)] +#[derive(Clone, Default)] +pub struct ConverterOptions { + /// Emit cleaner, more conformant Markdown (code-fence languages preserved, + /// no inline-run spacing artifacts) instead of docling's byte-for-byte + /// legacy output. Markdown only. Default `false`. + pub strict: Option, + /// For HTML/EPUB, resolve external `` (data: URIs, local files, + /// http(s) URLs, EPUB entries) and embed the bytes. Off by default; when on, + /// http(s) URLs are fetched over the network — enable only for trusted input. + pub fetch_images: Option, + /// Restrict the converter to these formats (ids like `"md"`, `"pdf"`, or + /// extensions like `".html"`); anything else is rejected. Default: accept all. + pub allowed_formats: Option>, +} + +/// Per-call output options (how to render the converted document). +#[napi(object)] +#[derive(Clone, Default)] +pub struct OutputOptions { + /// `"markdown"` (default) or `"json"` (docling-core DoclingDocument wire format). + pub to: Option, + /// Picture handling for Markdown: `"placeholder"` (default), `"embedded"` + /// (base64 data URIs inline), or `"referenced"` (returns image files in + /// `images`). Ignored for JSON, which always embeds images as data URIs. + pub image_mode: Option, + /// Directory name used in `referenced` image links. Default `"artifacts"`. + pub artifacts_dir: Option, +} + +/// All options for the one-shot module-level functions (converter config + +/// output options in a single object). +#[napi(object)] +#[derive(Clone, Default)] +pub struct ConvertOptions { + pub strict: Option, + pub fetch_images: Option, + pub allowed_formats: Option>, + pub to: Option, + pub image_mode: Option, + pub artifacts_dir: Option, +} + +/// In-memory input for [`DocumentConverter::convert`] / [`convert`]. +#[napi(object)] +pub struct ConvertInput { + /// Logical document name (used as the docling document name). + pub name: String, + /// Raw file bytes. + pub data: Buffer, + /// Format id or extension (e.g. `"md"`, `"pdf"`, `".html"`). Omit to infer + /// from an extension on `name`. + pub format: Option, +} + +/// One extracted image file, returned for the `referenced` image mode. +#[napi(object)] +pub struct ImageArtifact { + /// Path relative to the Markdown file (e.g. `"artifacts/image_000000.png"`). + pub path: String, + /// The image bytes to write at `path`. + pub data: Buffer, +} + +/// The result of a conversion. +#[napi(object)] +pub struct ConvertResult { + /// The rendered document: Markdown or JSON, per `to`. + pub content: String, + /// Detected input format id (e.g. `"md"`, `"pdf"`). + pub format: String, + /// `"success"`, `"partial_success"`, or `"failure"`. + pub status: String, + /// The document name. + pub input_name: String, + /// For the `referenced` image mode, the image files to write next to the + /// Markdown; empty otherwise. + pub images: Vec, +} + +// --------------------------------------------------------------------------- +// Internal, Send-safe conversion plumbing (shared by sync, async, streaming). +// --------------------------------------------------------------------------- + +/// Fully-resolved conversion config, free of any napi/JS types so it can move +/// onto a worker thread for the async and streaming paths. +struct ConvertConfig { + strict: bool, + fetch_images: bool, + allowed_formats: Option>, + to: OutputKind, + image_mode: ImageMode, + artifacts_dir: String, +} + +#[derive(Clone, Copy, PartialEq)] +enum OutputKind { + Markdown, + Json, +} + +/// A Send-safe conversion result (raw bytes, no `Buffer`), so it can be produced +/// off the JS thread and turned into a [`ConvertResult`] on resolve. Public only +/// because it is the `Output` of the public [`Task`] impls; not exposed to JS. +#[doc(hidden)] +pub struct RawResult { + content: String, + format: String, + status: String, + input_name: String, + images: Vec<(String, Vec)>, +} + +impl RawResult { + fn into_js(self) -> ConvertResult { + ConvertResult { + content: self.content, + format: self.format, + status: self.status, + input_name: self.input_name, + images: self + .images + .into_iter() + .map(|(path, data)| ImageArtifact { + path, + data: data.into(), + }) + .collect(), + } + } +} + +fn build_config( + strict: Option, + fetch_images: Option, + allowed_formats: Option>, + to: Option, + image_mode: Option, + artifacts_dir: Option, +) -> Result { + let allowed = match allowed_formats { + Some(list) => Some( + list.iter() + .map(|s| parse_format(s)) + .collect::>>()?, + ), + None => None, + }; + Ok(ConvertConfig { + strict: strict.unwrap_or(false), + fetch_images: fetch_images.unwrap_or(false), + allowed_formats: allowed, + to: parse_output_kind(to.as_deref())?, + image_mode: parse_image_mode(image_mode.as_deref())?, + artifacts_dir: artifacts_dir.unwrap_or_else(|| "artifacts".to_string()), + }) +} + +fn build_converter(cfg: &ConvertConfig) -> RsConverter { + let base = match &cfg.allowed_formats { + Some(list) => RsConverter::with_allowed_formats(list.iter().copied()), + None => RsConverter::new(), + }; + base.strict(cfg.strict).fetch_images(cfg.fetch_images) +} + +/// Render an already-converted document to Markdown/JSON per the config. The +/// document's `strict_markdown` is assumed already set by whoever produced it. +fn render_doc( + doc: DoclingDocument, + cfg: &ConvertConfig, + input_name: String, + format: String, + status: String, +) -> RawResult { + let (content, images) = match cfg.to { + OutputKind::Json => (doc.export_to_json(), Vec::new()), + OutputKind::Markdown => match cfg.image_mode { + ImageMode::Placeholder => (doc.export_to_markdown(), Vec::new()), + mode => doc.export_to_markdown_with_images(mode, &cfg.artifacts_dir), + }, + }; + RawResult { + content, + format, + status, + input_name, + images, + } +} + +/// Run a buffered conversion and render it per the config. Runs off the JS +/// thread for the async path, so it must stay free of napi/JS types. +fn run_convert(source: SourceDocument, cfg: &ConvertConfig) -> Result { + let converter = build_converter(cfg); + let result = converter.convert(source).map_err(convert_err)?; + let format = result.format.as_str().to_string(); + let status = status_str(result.status); + Ok(render_doc( + result.document, + cfg, + result.input_name, + format, + status, + )) +} + +/// Load a [`SourceDocument`] from an in-memory [`ConvertInput`]. +fn source_from_input(input: ConvertInput) -> Result { + let format = match &input.format { + Some(f) => parse_format(f)?, + None => infer_format(&input.name).ok_or_else(|| { + Error::new( + Status::InvalidArg, + format!( + "could not infer a format from name '{}'; pass `format` explicitly", + input.name + ), + ) + })?, + }; + Ok(SourceDocument::from_bytes( + input.name, + format, + input.data.to_vec(), + )) +} + +// --------------------------------------------------------------------------- +// Module-level one-shot API. +// --------------------------------------------------------------------------- + +/// Convert a file on disk. Detects the format from the extension and (for +/// HTML/EPUB image fetching) resolves relative `` against the file's +/// directory. +#[napi] +pub fn convert_file(path: String, options: Option) -> Result { + let o = options.unwrap_or_default(); + let cfg = build_config( + o.strict, + o.fetch_images, + o.allowed_formats, + o.to, + o.image_mode, + o.artifacts_dir, + )?; + let source = SourceDocument::from_file(&path).map_err(convert_err)?; + Ok(run_convert(source, &cfg)?.into_js()) +} + +/// Convert in-memory bytes. +#[napi] +pub fn convert(input: ConvertInput, options: Option) -> Result { + let o = options.unwrap_or_default(); + let cfg = build_config( + o.strict, + o.fetch_images, + o.allowed_formats, + o.to, + o.image_mode, + o.artifacts_dir, + )?; + let source = source_from_input(input)?; + Ok(run_convert(source, &cfg)?.into_js()) +} + +/// Async (Promise-returning) [`convert_file`]. The CPU-bound work runs on the +/// libuv thread pool, keeping the event loop free — use this for PDF/image. +#[napi(ts_return_type = "Promise")] +pub fn convert_file_async( + path: String, + options: Option, +) -> Result> { + let o = options.unwrap_or_default(); + let cfg = build_config( + o.strict, + o.fetch_images, + o.allowed_formats, + o.to, + o.image_mode, + o.artifacts_dir, + )?; + Ok(AsyncTask::new(ConvertFileTask { path, cfg })) +} + +/// Async (Promise-returning) [`convert`]. +#[napi(ts_return_type = "Promise")] +pub fn convert_async( + input: ConvertInput, + options: Option, +) -> Result> { + let o = options.unwrap_or_default(); + let cfg = build_config( + o.strict, + o.fetch_images, + o.allowed_formats, + o.to, + o.image_mode, + o.artifacts_dir, + )?; + let source = source_from_input(input)?; + Ok(AsyncTask::new(ConvertBytesTask { + source: Some(source), + cfg, + })) +} + +pub struct ConvertFileTask { + path: String, + cfg: ConvertConfig, +} + +impl Task for ConvertFileTask { + type Output = RawResult; + type JsValue = ConvertResult; + + fn compute(&mut self) -> Result { + let source = SourceDocument::from_file(&self.path).map_err(convert_err)?; + run_convert(source, &self.cfg) + } + + fn resolve(&mut self, _env: Env, output: RawResult) -> Result { + Ok(output.into_js()) + } +} + +pub struct ConvertBytesTask { + // `Option` so `compute` can take ownership of the (non-Copy) source. + source: Option, + cfg: ConvertConfig, +} + +impl Task for ConvertBytesTask { + type Output = RawResult; + type JsValue = ConvertResult; + + fn compute(&mut self) -> Result { + let source = self + .source + .take() + .ok_or_else(|| Error::new(Status::GenericFailure, "conversion task reused"))?; + run_convert(source, &self.cfg) + } + + fn resolve(&mut self, _env: Env, output: RawResult) -> Result { + Ok(output.into_js()) + } +} + +// --------------------------------------------------------------------------- +// Reusable converter class. +// --------------------------------------------------------------------------- + +/// A reusable converter. Holds config (strict / fetch-images / allowed formats) +/// so you can convert many documents without re-parsing options each time — +/// the analogue of the Rust `DocumentConverter`. +#[napi] +pub struct DocumentConverter { + strict: bool, + fetch_images: bool, + allowed_formats: Option>, +} + +#[napi] +impl DocumentConverter { + #[napi(constructor)] + pub fn new(options: Option) -> Result { + let o = options.unwrap_or_default(); + let allowed = match o.allowed_formats { + Some(list) => Some( + list.iter() + .map(|s| parse_format(s)) + .collect::>>()?, + ), + None => None, + }; + Ok(Self { + strict: o.strict.unwrap_or(false), + fetch_images: o.fetch_images.unwrap_or(false), + allowed_formats: allowed, + }) + } + + fn config(&self, out: Option) -> Result { + let out = out.unwrap_or_default(); + Ok(ConvertConfig { + strict: self.strict, + fetch_images: self.fetch_images, + allowed_formats: self.allowed_formats.clone(), + to: parse_output_kind(out.to.as_deref())?, + image_mode: parse_image_mode(out.image_mode.as_deref())?, + artifacts_dir: out.artifacts_dir.unwrap_or_else(|| "artifacts".to_string()), + }) + } + + /// Convert a file on disk (sync). + #[napi] + pub fn convert_file( + &self, + path: String, + options: Option, + ) -> Result { + let cfg = self.config(options)?; + let source = SourceDocument::from_file(&path).map_err(convert_err)?; + Ok(run_convert(source, &cfg)?.into_js()) + } + + /// Convert in-memory bytes (sync). + #[napi] + pub fn convert( + &self, + input: ConvertInput, + options: Option, + ) -> Result { + let cfg = self.config(options)?; + let source = source_from_input(input)?; + Ok(run_convert(source, &cfg)?.into_js()) + } + + /// Async (Promise-returning) file conversion (runs off the event loop). + #[napi(ts_return_type = "Promise")] + pub fn convert_file_async( + &self, + path: String, + options: Option, + ) -> Result> { + let cfg = self.config(options)?; + Ok(AsyncTask::new(ConvertFileTask { path, cfg })) + } + + /// Async (Promise-returning) bytes conversion (runs off the event loop). + #[napi(ts_return_type = "Promise")] + pub fn convert_async( + &self, + input: ConvertInput, + options: Option, + ) -> Result> { + let cfg = self.config(options)?; + let source = source_from_input(input)?; + Ok(AsyncTask::new(ConvertBytesTask { + source: Some(source), + cfg, + })) + } + + /// Stream a file's Markdown in chunks, in document order, as conversion + /// progresses (the headline win for PDF, whose pages convert in parallel). + /// + /// `callback` is invoked as `(err, chunk)`: once per Markdown chunk with + /// `chunk` a string, once with `chunk === null` at the end, or once with a + /// non-null `err` on failure. Only `placeholder` / `embedded` image modes + /// stream; `referenced` is rejected. Prefer the `streamFileMarkdown` + /// async-generator wrapper in JS over calling this directly. + #[napi] + pub fn convert_file_streaming( + &self, + path: String, + callback: ThreadsafeFunction, ErrorStrategy::CalleeHandled>, + options: Option, + ) -> Result<()> { + let cfg = self.config(options)?; + let converter = build_converter(&cfg); + let image_mode = cfg.image_mode; + // The background conversion thread owns the stream and pushes each chunk + // through the threadsafe function (which marshals back to the JS loop). + std::thread::spawn(move || { + let source = match SourceDocument::from_file(&path).map_err(convert_err) { + Ok(s) => s, + Err(e) => { + callback.call(Err(e), ThreadsafeFunctionCallMode::NonBlocking); + return; + } + }; + let stream = match converter.convert_streaming_images(source, image_mode) { + Ok(s) => s, + Err(e) => { + callback.call(Err(convert_err(e)), ThreadsafeFunctionCallMode::NonBlocking); + return; + } + }; + for chunk in stream { + match chunk { + Ok(s) => { + callback.call(Ok(Some(s)), ThreadsafeFunctionCallMode::NonBlocking); + } + Err(e) => { + callback.call(Err(convert_err(e)), ThreadsafeFunctionCallMode::NonBlocking); + return; + } + } + } + // End-of-stream sentinel. + callback.call(Ok(None), ThreadsafeFunctionCallMode::NonBlocking); + }); + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Reusable warm PDF/image pipeline. +// --------------------------------------------------------------------------- + +/// A reusable PDF/image pipeline that keeps the ONNX models (layout, OCR, +/// TableFormer) loaded across calls — the analogue of the Rust `Pipeline`. Use +/// this instead of the per-call `convertFile` when converting many PDFs/images: +/// the one-shot functions rebuild the pipeline (reloading every model) each +/// call, whereas this loads them once. +/// +/// Handles `pdf` and `image` inputs (the ML pipeline). Models load lazily on +/// first use, so constructing a `Pipeline` is cheap; the first conversion pays +/// the model-load cost. Synchronous and single-threaded — reuse one instance +/// for a sequence of documents (e.g. behind a job queue). +#[napi] +pub struct Pipeline { + // RefCell: the Rust pipeline needs `&mut` to convert (models are mutable + // sessions), but napi hands us `&self`. A Pipeline is bound to the JS thread, + // so single-threaded interior mutability is sound here. + inner: RefCell, + strict: bool, +} + +#[napi] +impl Pipeline { + /// Construct the pipeline. Only `strict` is read (cleaner Markdown); + /// `fetchImages` / `allowedFormats` don't apply to the PDF/image pipeline. + #[napi(constructor)] + pub fn new(options: Option) -> Result { + let strict = options.and_then(|o| o.strict).unwrap_or(false); + Ok(Self { + inner: RefCell::new(RsPipeline::new().map_err(convert_err)?), + strict, + }) + } + + /// Convert a PDF or image file, reusing the warm models. + #[napi] + pub fn convert_file( + &self, + path: String, + options: Option, + ) -> Result { + let source = SourceDocument::from_file(&path).map_err(convert_err)?; + self.run(source, options) + } + + /// Convert PDF or image bytes, reusing the warm models. + #[napi] + pub fn convert( + &self, + input: ConvertInput, + options: Option, + ) -> Result { + let source = source_from_input(input)?; + self.run(source, options) + } +} + +impl Pipeline { + fn run(&self, source: SourceDocument, options: Option) -> Result { + let cfg = output_config(options, self.strict)?; + let mut pipe = self.inner.borrow_mut(); + let mut doc = match source.format { + InputFormat::Pdf => pipe + .convert(&source.bytes, None, &source.name) + .map_err(convert_err)?, + InputFormat::Image => pipe + .convert_image(&source.bytes, &source.name) + .map_err(convert_err)?, + other => { + return Err(Error::new( + Status::InvalidArg, + format!( + "Pipeline handles pdf and image inputs (the ML pipeline); got '{}'. \ + Use convertFile / convert for other formats.", + other.as_str() + ), + )) + } + }; + doc.strict_markdown = self.strict; + Ok(render_doc( + doc, + &cfg, + source.name, + source.format.as_str().to_string(), + "success".to_string(), + ) + .into_js()) + } +} + +/// Build a render-only [`ConvertConfig`] from per-call output options (the +/// converter-config fields are unused when rendering a document we already have). +fn output_config(out: Option, strict: bool) -> Result { + let out = out.unwrap_or_default(); + Ok(ConvertConfig { + strict, + fetch_images: false, + allowed_formats: None, + to: parse_output_kind(out.to.as_deref())?, + image_mode: parse_image_mode(out.image_mode.as_deref())?, + artifacts_dir: out.artifacts_dir.unwrap_or_else(|| "artifacts".to_string()), + }) +} + +// --------------------------------------------------------------------------- +// Format helpers exposed to JS. +// --------------------------------------------------------------------------- + +/// The list of supported input format ids. +#[napi] +pub fn supported_formats() -> Vec { + [ + "docx", + "pptx", + "html", + "image", + "pdf", + "asciidoc", + "md", + "csv", + "xlsx", + "odt", + "ods", + "odp", + "xml_uspto", + "xml_jats", + "xml_xbrl", + "mets_gbs", + "json_docling", + "vtt", + "latex", + "email", + "epub", + ] + .iter() + .map(|s| s.to_string()) + .collect() +} + +/// Detect a format id from a filename or extension (e.g. `"report.pdf"` → +/// `"pdf"`). Returns `null` for unknown extensions. +#[napi] +pub fn format_from_name(name: String) -> Option { + infer_format(&name).map(|f| f.as_str().to_string()) +} + +// --------------------------------------------------------------------------- +// Non-exported helpers. +// --------------------------------------------------------------------------- + +fn infer_format(name: &str) -> Option { + let ext = name.rsplit('.').next().filter(|e| *e != name)?; + InputFormat::from_extension(ext) +} + +fn parse_output_kind(to: Option<&str>) -> Result { + match to.map(str::to_ascii_lowercase).as_deref() { + None | Some("md") | Some("markdown") => Ok(OutputKind::Markdown), + Some("json") => Ok(OutputKind::Json), + Some(other) => Err(Error::new( + Status::InvalidArg, + format!("unknown `to` '{other}' (expected: markdown, json)"), + )), + } +} + +fn parse_image_mode(mode: Option<&str>) -> Result { + match mode.map(str::to_ascii_lowercase).as_deref() { + None | Some("placeholder") => Ok(ImageMode::Placeholder), + Some("embedded") => Ok(ImageMode::Embedded), + Some("referenced") => Ok(ImageMode::Referenced), + Some(other) => Err(Error::new( + Status::InvalidArg, + format!("unknown imageMode '{other}' (expected: placeholder, embedded, referenced)"), + )), + } +} + +/// Resolve a user-supplied format string — a format id (as reported by +/// [`supported_formats`]) or a file extension — to an [`InputFormat`]. +fn parse_format(s: &str) -> Result { + let key = s.trim().trim_start_matches('.').to_ascii_lowercase(); + // Extensions first (covers ".html", "jpg", "eml", …); then format ids for + // the ones extensions don't name (e.g. "image", "xml_uspto"). + if let Some(f) = InputFormat::from_extension(&key) { + return Ok(f); + } + let f = match key.as_str() { + "image" => InputFormat::Image, + "asciidoc" => InputFormat::Asciidoc, + "markdown" => InputFormat::Md, + "xml_uspto" | "uspto" => InputFormat::XmlUspto, + "xml_jats" | "jats" => InputFormat::XmlJats, + "xml_xbrl" | "xbrl" => InputFormat::XmlXbrl, + "json_docling" => InputFormat::JsonDocling, + "mets_gbs" => InputFormat::MetsGbs, + "email" => InputFormat::Email, + "latex" => InputFormat::Latex, + _ => { + return Err(Error::new( + Status::InvalidArg, + format!("unknown format '{s}'"), + )) + } + }; + Ok(f) +} + +fn status_str(status: ConversionStatus) -> String { + match status { + ConversionStatus::Success => "success", + ConversionStatus::PartialSuccess => "partial_success", + ConversionStatus::Failure => "failure", + } + .to_string() +} + +fn convert_err(e: impl std::fmt::Display) -> Error { + Error::new(Status::GenericFailure, e.to_string()) +} diff --git a/crates/fleischwolf-node/test/smoke.mjs b/crates/fleischwolf-node/test/smoke.mjs new file mode 100644 index 00000000..b5562d13 --- /dev/null +++ b/crates/fleischwolf-node/test/smoke.mjs @@ -0,0 +1,155 @@ +// Minimal smoke test exercising every part of the binding: sync + async +// convert, in-memory bytes, JSON output, the reusable class, streaming, and the +// format helpers. Run with `node test/smoke.mjs` (or `bun test/smoke.mjs`). +// +// Exits non-zero on the first failed assertion, so it doubles as a CI check. + +import assert from 'node:assert/strict' +import { + checkDependencies, + convert, + convertFile, + convertFileAsync, + DocumentConverter, + formatFromName, + Pipeline, + streamFileMarkdown, + supportedFormats, +} from '../index.js' + +let passed = 0 +const check = (name, fn) => { + return Promise.resolve() + .then(fn) + .then(() => { + passed++ + console.log(` ok ${name}`) + }) + .catch((err) => { + console.error(`fail ${name}\n ${err.message}`) + process.exitCode = 1 + throw err + }) +} + +const MD = '# Title\n\nHello **world**.\n\n- one\n- two\n' + +async function main() { + await check('supportedFormats lists md and pdf', () => { + const formats = supportedFormats() + assert.ok(formats.includes('md')) + assert.ok(formats.includes('pdf')) + }) + + await check('formatFromName detects extensions', () => { + assert.equal(formatFromName('report.pdf'), 'pdf') + assert.equal(formatFromName('page.html'), 'html') + assert.equal(formatFromName('mystery.zzz'), null) + }) + + await check('convert (bytes) → Markdown round-trips', () => { + const res = convert({ name: 'doc', data: Buffer.from(MD), format: 'md' }) + assert.equal(res.status, 'success') + assert.equal(res.format, 'md') + assert.match(res.content, /# Title/) + assert.match(res.content, /Hello/) + }) + + await check('convert (bytes) → JSON is docling-core wire format', () => { + const res = convert({ name: 'doc', data: Buffer.from(MD), format: 'md' }, { to: 'json' }) + const doc = JSON.parse(res.content) + assert.equal(doc.schema_name, 'DoclingDocument') + assert.ok(Array.isArray(doc.texts)) + }) + + await check('format inferred from name when omitted', () => { + const res = convert({ name: 'notes.md', data: Buffer.from(MD) }) + assert.equal(res.format, 'md') + }) + + await check('DocumentConverter class is reusable', () => { + const converter = new DocumentConverter({ strict: true }) + const a = converter.convert({ name: 'a.md', data: Buffer.from('# A\n') }) + const b = converter.convert({ name: 'b.md', data: Buffer.from('# B\n') }) + assert.match(a.content, /# A/) + assert.match(b.content, /# B/) + }) + + await check('allowedFormats rejects other formats', () => { + const converter = new DocumentConverter({ allowedFormats: ['csv'] }) + assert.throws(() => converter.convert({ name: 'x.md', data: Buffer.from(MD) })) + }) + + await check('unknown format string is rejected', () => { + assert.throws(() => convert({ name: 'x', data: Buffer.from(MD), format: 'nope' })) + }) + + // --- ML dependency guards (models not installed in this test env) --------- + + await check('checkDependencies reports status without downloading', () => { + const status = checkDependencies() + assert.equal(typeof status.ready, 'boolean') + assert.equal(typeof status.pdfium, 'boolean') + assert.ok(Array.isArray(status.missing)) + }) + + // These assume the ML models are NOT installed (true on a fresh CI checkout). + const depsInstalled = checkDependencies().ready + if (!depsInstalled) { + await check('convert PDF (sync) throws pointing at installDependencies', () => { + assert.throws( + () => convert({ name: 'doc.pdf', data: Buffer.from('%PDF-1.4') }), + /installDependencies/, + ) + }) + + await check('convertFileAsync PDF rejects (not a sync throw)', async () => { + await assert.rejects(convertFileAsync('missing.pdf'), /installDependencies/) + }) + + await check('image bytes are guarded too', () => { + assert.throws(() => convert({ name: 'scan.png', data: Buffer.from([0]) }), /installDependencies/) + }) + + await check('Pipeline convertFile is guarded', () => { + const pipe = new Pipeline() + assert.throws(() => pipe.convertFile('x.pdf'), /installDependencies/) + }) + } else { + console.log(' -- ML deps installed; skipping guard checks') + } + + // File-based sync + async + streaming, using a temp Markdown file. + const { writeFileSync, mkdtempSync } = await import('node:fs') + const { tmpdir } = await import('node:os') + const { join } = await import('node:path') + const dir = mkdtempSync(join(tmpdir(), 'fw-smoke-')) + const file = join(dir, 'doc.md') + writeFileSync(file, MD) + + await check('convertFile (sync) reads from disk', () => { + const res = convertFile(file) + assert.match(res.content, /# Title/) + assert.equal(res.inputName, 'doc') + }) + + await check('convertFileAsync returns a Promise', async () => { + const res = await convertFileAsync(file, { to: 'json' }) + assert.equal(JSON.parse(res.content).schema_name, 'DoclingDocument') + }) + + await check('streamFileMarkdown yields chunks equal to buffered output', async () => { + let streamed = '' + for await (const chunk of streamFileMarkdown(file)) { + streamed += chunk + } + assert.equal(streamed, convertFile(file).content) + assert.ok(streamed.length > 0) + }) + + console.log(`\n${passed} checks passed`) +} + +main().catch(() => { + process.exitCode = 1 +})