fix(pdf): align OCR text layer with the scanned image

app/models/OCRDocument.ts

@@ -27,6 +27,7 @@ import {
     EVENT_FOLDER_ADDED,
     EVENT_FOLDER_UPDATED,
     IMG_FORMAT,
+    OCR_ITERATOR_LEVEL,
     SEPARATOR,
     SETTINGS_DOCUMENT_NAME_FORMAT,
     getImageExportSettings
@@ -416,6 +417,7 @@ export class OCRDocument extends Observable implements Document {
                 rotation: page.rotation,
                 // oem: 0,
                 detectContours: 0,
+                iteratorLevel: OCR_ITERATOR_LEVEL,
                 trim: false
             },
             onProgress

app/services/ocr.ts

@@ -4,6 +4,7 @@ import { NoNetworkError } from '@akylas/nativescript-app-utils/error';
 import { ocrDocumentFromFile } from 'plugin-nativeprocessor';
 import type { OCRDocument } from '~/models/OCRDocument';
 import { networkService, wrapNativeHttpException } from '~/services/api';
+import { OCR_ITERATOR_LEVEL } from '~/utils/constants';
 
 export const OCRLanguages = {
     afr: 'Afrikaans',
@@ -268,6 +269,7 @@ export class OCRService extends Observable {
                 // rotation: page.rotation,
                 // oem: 0,
                 detectContours: 0,
+                iteratorLevel: OCR_ITERATOR_LEVEL,
                 trim: false
             },
             onProgress

app/services/pdf/PDFCanvas.ts

@@ -1,5 +1,5 @@
 import { DeviceContext } from '@nativescript-community/sentry/integrations';
-import { Canvas, ColorMatrixColorFilter, LayoutAlignment, Paint, StaticLayout } from '@nativescript-community/ui-canvas';
+import { Canvas, ColorMatrixColorFilter, Paint } from '@nativescript-community/ui-canvas';
 import { ApplicationSettings, Screen, Utils } from '@nativescript/core';
 import { getActualLanguage } from '@shared/helpers/lang';
 import type { OCRDocument, OCRPage } from '~/models/OCRDocument';
@@ -53,6 +53,14 @@ const bgPaint = new Paint();
 bgPaint.color = 'white';
 bgPaint.setShadowLayer(6, 0, 2, '#00000088');
 
+// OCR text layer geometry. A tesseract line box spans ascenders to descenders:
+// the font size is the box height and the baseline sits just above the box bottom.
+// Keep in sync with PDFUtils.kt (android).
+const OCR_FONT_SIZE_RATIO = 1;
+const OCR_BASELINE_RATIO = 0.8;
+const OCR_MIN_HORIZONTAL_SCALE = 0.25;
+const OCR_MAX_HORIZONTAL_SCALE = 4;
+
 function ptToPixel(value, dpi) {
     //1pt = 1/72 inch
     //1inch = dpi pixels
@@ -155,7 +163,6 @@ export default class PDFCanvas {
             return;
         }
 
-        const textScale = Screen.mainScreen.scale * (__IOS__ ? 2.2 : 1.4);
         const src = page.imagePath;
         let imageWidth = page.width;
         let imageHeight = page.height;
@@ -191,17 +198,36 @@ export default class PDFCanvas {
         recycleImages(image);
 
         if (this.options.draw_ocr_text && page.ocrData) {
-            const ocrScale = toDrawWidth / page.ocrData.imageWidth;
-            canvas.scale(ocrScale, ocrScale, 0, 0);
+            const ocrData = page.ocrData;
+            // ocr boxes live in the ocr image space, which is already rotated (the ocr ran with page.rotation)
+            const scaleX = toDrawWidth / ocrData.imageWidth;
+            const scaleY = toDrawHeight / ocrData.imageHeight;
             textPaint.color = !PRODUCTION && DEV_LOG ? '#ff000088' : '#ffffff01';
-            page.ocrData.blocks.forEach((block) => {
-                canvas.save();
-                // TODO: understand why that kind of scale is necessary
-                textPaint.textSize = (block.fontSize || 16) * textScale;
-                const staticLayout = new StaticLayout(block.text, textPaint, block.box.width, LayoutAlignment.ALIGN_NORMAL, 1, 0, true);
-                canvas.translate(block.box.x, block.box.y);
-                staticLayout.draw(canvas);
-                canvas.restore();
+            ocrData.blocks.forEach((block) => {
+                // a block holds several lines when the ocr ran at paragraph level (documents
+                // scanned before line level ocr): share the box height between them
+                const lines = block.text
+                    .split('\n')
+                    .map((line) => line.trim())
+                    .filter((line) => line.length > 0);
+                if (lines.length === 0) {
+                    return;
+                }
+                const boxWidth = block.box.width * scaleX;
+                const boxTop = block.box.y * scaleY;
+                const lineHeight = (block.box.height * scaleY) / lines.length;
+                textPaint.textSize = lineHeight * OCR_FONT_SIZE_RATIO;
+                lines.forEach((line, lineIndex) => {
+                    const measured = textPaint.measureText(line);
+                    // stretch the run so that it covers exactly the detected box
+                    const horizontalScale = measured > 0 ? Math.min(Math.max(boxWidth / measured, OCR_MIN_HORIZONTAL_SCALE), OCR_MAX_HORIZONTAL_SCALE) : 1;
+                    canvas.save();
+                    // canvas is y down here: the baseline sits below the line top
+                    canvas.translate(block.box.x * scaleX, boxTop + (lineIndex + OCR_BASELINE_RATIO) * lineHeight);
+                    canvas.scale(horizontalScale, 1, 0, 0);
+                    canvas.drawText(line, 0, 0, textPaint);
+                    canvas.restore();
+                });
             });
         }
     }

app/services/pdf/PDFExportCanvas.android.ts

@@ -1,4 +1,4 @@
-import { Screen, Utils, knownFolders } from '@nativescript/core';
+import { Utils, knownFolders } from '@nativescript/core';
 import { PDF_EXT } from '~/utils/constants';
 import { getColorMatrix } from '~/utils/matrix';
 import { PDFExportOptions } from './PDFCanvas';
@@ -13,7 +13,7 @@ export default class PDFExportCanvas extends PDFExportCanvasBase {
                 page.colorMatrix = getColorMatrix(page.colorType);
             }
         });
-        const options = JSON.stringify({ ...this.options, text_scale: Screen.mainScreen.scale * 1.4, pages });
+        const options = JSON.stringify({ ...this.options, pages });
         DEV_LOG && console.log('PDFExportCanvas', 'export', folder, filename, compress, options);
         const outputPath = com.akylas.documentscanner.utils.PDFUtils.Companion.generatePDF(Utils.android.getApplicationContext(), folder, filename, options);
         DEV_LOG && console.log('PDFExportCanvas', 'export done', JSON.stringify(this.options), options.length, Date.now() - start, 'ms');

app/services/pdf/PDFExporter.ts

@@ -1,5 +1,5 @@
 import { WorkerEventType } from '@akylas/nativescript-app-utils/worker/BaseWorker';
-import { Screen, Utils, knownFolders, path } from '@nativescript/core';
+import { Utils, knownFolders, path } from '@nativescript/core';
 import { wrapNativeException } from '@nativescript/core/utils';
 import { getActualLanguage } from '@shared/helpers/lang';
 import { CustomError, SilentError, TimeoutError } from '@akylas/nativescript-app-utils/error';
@@ -47,7 +47,6 @@ export async function exportPDFAsync({ compress, document, filename, folder, opt
         const options = JSON.stringify({
             ...defaultOptions,
             // page_padding: Utils.layout.toDevicePixels(pdfCanvas.options.page_padding),
-            text_scale: Screen.mainScreen.scale * 1.4,
             pages: pages.map((p) => ({ ...p.page, colorMatrix: getPageColorMatrix(p.page, black_white ? 'grayscale' : undefined) })),
             ...(baseOptions ? baseOptions : {}),
             debug: false

app/utils/constants.ts

@@ -102,6 +102,9 @@ export const DEFAULT_VIEW_STYLE = CARD_APP ? 'columns' : 'default';
 export const DEFAULT_FOLDER_VIEW_STYLE = 'horizontal';
 export const DEFAULT_FORCE_WHITE_BACKGROUND_QRCODE = false;
 export const DEFAULT_OCR_COPY_USE_SPACE = false;
+// tesseract PageIteratorLevel.RIL_TEXTLINE: one OCR block per text line.
+// Line boxes are what makes the PDF text layer land on the raster text.
+export const OCR_ITERATOR_LEVEL = 2;
 export const DEFAULT_PDF_OPTIONS = {
     paper_size: 'full',
     color: 'color',