diff --git a/app/models/OCRDocument.ts b/app/models/OCRDocument.ts index 3ece22c4..ca22e928 100644 --- a/app/models/OCRDocument.ts +++ b/app/models/OCRDocument.ts @@ -27,6 +27,7 @@ import { EVENT_FOLDER_ADDED, EVENT_FOLDER_UPDATED, IMG_FORMAT, + OCR_ITERATOR_LEVEL, SEPARATOR, SETTINGS_DOCUMENT_NAME_FORMAT, getImageExportSettings @@ -416,6 +417,7 @@ export class OCRDocument extends Observable implements Document { rotation: page.rotation, // oem: 0, detectContours: 0, + iteratorLevel: OCR_ITERATOR_LEVEL, trim: false }, onProgress diff --git a/app/services/ocr.ts b/app/services/ocr.ts index 3183e8a6..c66e7c48 100644 --- a/app/services/ocr.ts +++ b/app/services/ocr.ts @@ -4,6 +4,7 @@ import { NoNetworkError } from '@akylas/nativescript-app-utils/error'; import { ocrDocumentFromFile } from 'plugin-nativeprocessor'; import type { OCRDocument } from '~/models/OCRDocument'; import { networkService, wrapNativeHttpException } from '~/services/api'; +import { OCR_ITERATOR_LEVEL } from '~/utils/constants'; export const OCRLanguages = { afr: 'Afrikaans', @@ -268,6 +269,7 @@ export class OCRService extends Observable { // rotation: page.rotation, // oem: 0, detectContours: 0, + iteratorLevel: OCR_ITERATOR_LEVEL, trim: false }, onProgress diff --git a/app/services/pdf/PDFCanvas.ts b/app/services/pdf/PDFCanvas.ts index 92cd4e6b..93640ec9 100644 --- a/app/services/pdf/PDFCanvas.ts +++ b/app/services/pdf/PDFCanvas.ts @@ -1,5 +1,5 @@ import { DeviceContext } from '@nativescript-community/sentry/integrations'; -import { Canvas, ColorMatrixColorFilter, LayoutAlignment, Paint, StaticLayout } from '@nativescript-community/ui-canvas'; +import { Canvas, ColorMatrixColorFilter, Paint } from '@nativescript-community/ui-canvas'; import { ApplicationSettings, Screen, Utils } from '@nativescript/core'; import { getActualLanguage } from '@shared/helpers/lang'; import type { OCRDocument, OCRPage } from '~/models/OCRDocument'; @@ -53,6 +53,14 @@ const bgPaint = new Paint(); bgPaint.color = 'white'; bgPaint.setShadowLayer(6, 0, 2, '#00000088'); +// OCR text layer geometry. A tesseract line box spans ascenders to descenders: +// the font size is the box height and the baseline sits just above the box bottom. +// Keep in sync with PDFUtils.kt (android). +const OCR_FONT_SIZE_RATIO = 1; +const OCR_BASELINE_RATIO = 0.8; +const OCR_MIN_HORIZONTAL_SCALE = 0.25; +const OCR_MAX_HORIZONTAL_SCALE = 4; + function ptToPixel(value, dpi) { //1pt = 1/72 inch //1inch = dpi pixels @@ -155,7 +163,6 @@ export default class PDFCanvas { return; } - const textScale = Screen.mainScreen.scale * (__IOS__ ? 2.2 : 1.4); const src = page.imagePath; let imageWidth = page.width; let imageHeight = page.height; @@ -191,17 +198,36 @@ export default class PDFCanvas { recycleImages(image); if (this.options.draw_ocr_text && page.ocrData) { - const ocrScale = toDrawWidth / page.ocrData.imageWidth; - canvas.scale(ocrScale, ocrScale, 0, 0); + const ocrData = page.ocrData; + // ocr boxes live in the ocr image space, which is already rotated (the ocr ran with page.rotation) + const scaleX = toDrawWidth / ocrData.imageWidth; + const scaleY = toDrawHeight / ocrData.imageHeight; textPaint.color = !PRODUCTION && DEV_LOG ? '#ff000088' : '#ffffff01'; - page.ocrData.blocks.forEach((block) => { - canvas.save(); - // TODO: understand why that kind of scale is necessary - textPaint.textSize = (block.fontSize || 16) * textScale; - const staticLayout = new StaticLayout(block.text, textPaint, block.box.width, LayoutAlignment.ALIGN_NORMAL, 1, 0, true); - canvas.translate(block.box.x, block.box.y); - staticLayout.draw(canvas); - canvas.restore(); + ocrData.blocks.forEach((block) => { + // a block holds several lines when the ocr ran at paragraph level (documents + // scanned before line level ocr): share the box height between them + const lines = block.text + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0); + if (lines.length === 0) { + return; + } + const boxWidth = block.box.width * scaleX; + const boxTop = block.box.y * scaleY; + const lineHeight = (block.box.height * scaleY) / lines.length; + textPaint.textSize = lineHeight * OCR_FONT_SIZE_RATIO; + lines.forEach((line, lineIndex) => { + const measured = textPaint.measureText(line); + // stretch the run so that it covers exactly the detected box + const horizontalScale = measured > 0 ? Math.min(Math.max(boxWidth / measured, OCR_MIN_HORIZONTAL_SCALE), OCR_MAX_HORIZONTAL_SCALE) : 1; + canvas.save(); + // canvas is y down here: the baseline sits below the line top + canvas.translate(block.box.x * scaleX, boxTop + (lineIndex + OCR_BASELINE_RATIO) * lineHeight); + canvas.scale(horizontalScale, 1, 0, 0); + canvas.drawText(line, 0, 0, textPaint); + canvas.restore(); + }); }); } } diff --git a/app/services/pdf/PDFExportCanvas.android.ts b/app/services/pdf/PDFExportCanvas.android.ts index ac4bee02..3c41b9db 100644 --- a/app/services/pdf/PDFExportCanvas.android.ts +++ b/app/services/pdf/PDFExportCanvas.android.ts @@ -1,4 +1,4 @@ -import { Screen, Utils, knownFolders } from '@nativescript/core'; +import { Utils, knownFolders } from '@nativescript/core'; import { PDF_EXT } from '~/utils/constants'; import { getColorMatrix } from '~/utils/matrix'; import { PDFExportOptions } from './PDFCanvas'; @@ -13,7 +13,7 @@ export default class PDFExportCanvas extends PDFExportCanvasBase { page.colorMatrix = getColorMatrix(page.colorType); } }); - const options = JSON.stringify({ ...this.options, text_scale: Screen.mainScreen.scale * 1.4, pages }); + const options = JSON.stringify({ ...this.options, pages }); DEV_LOG && console.log('PDFExportCanvas', 'export', folder, filename, compress, options); const outputPath = com.akylas.documentscanner.utils.PDFUtils.Companion.generatePDF(Utils.android.getApplicationContext(), folder, filename, options); DEV_LOG && console.log('PDFExportCanvas', 'export done', JSON.stringify(this.options), options.length, Date.now() - start, 'ms'); diff --git a/app/services/pdf/PDFExporter.ts b/app/services/pdf/PDFExporter.ts index 8c17721f..75345f1f 100644 --- a/app/services/pdf/PDFExporter.ts +++ b/app/services/pdf/PDFExporter.ts @@ -1,5 +1,5 @@ import { WorkerEventType } from '@akylas/nativescript-app-utils/worker/BaseWorker'; -import { Screen, Utils, knownFolders, path } from '@nativescript/core'; +import { Utils, knownFolders, path } from '@nativescript/core'; import { wrapNativeException } from '@nativescript/core/utils'; import { getActualLanguage } from '@shared/helpers/lang'; import { CustomError, SilentError, TimeoutError } from '@akylas/nativescript-app-utils/error'; @@ -47,7 +47,6 @@ export async function exportPDFAsync({ compress, document, filename, folder, opt const options = JSON.stringify({ ...defaultOptions, // page_padding: Utils.layout.toDevicePixels(pdfCanvas.options.page_padding), - text_scale: Screen.mainScreen.scale * 1.4, pages: pages.map((p) => ({ ...p.page, colorMatrix: getPageColorMatrix(p.page, black_white ? 'grayscale' : undefined) })), ...(baseOptions ? baseOptions : {}), debug: false diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 201a5d51..d6a36ed1 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -102,6 +102,9 @@ export const DEFAULT_VIEW_STYLE = CARD_APP ? 'columns' : 'default'; export const DEFAULT_FOLDER_VIEW_STYLE = 'horizontal'; export const DEFAULT_FORCE_WHITE_BACKGROUND_QRCODE = false; export const DEFAULT_OCR_COPY_USE_SPACE = false; +// tesseract PageIteratorLevel.RIL_TEXTLINE: one OCR block per text line. +// Line boxes are what makes the PDF text layer land on the raster text. +export const OCR_ITERATOR_LEVEL = 2; export const DEFAULT_PDF_OPTIONS = { paper_size: 'full', color: 'color', diff --git a/plugin-nativeprocessor/platforms/android/java/com/akylas/documentscanner/utils/PDFUtils.kt b/plugin-nativeprocessor/platforms/android/java/com/akylas/documentscanner/utils/PDFUtils.kt index 12d7d718..425fe683 100644 --- a/plugin-nativeprocessor/platforms/android/java/com/akylas/documentscanner/utils/PDFUtils.kt +++ b/plugin-nativeprocessor/platforms/android/java/com/akylas/documentscanner/utils/PDFUtils.kt @@ -20,7 +20,6 @@ import android.print.PrintManager import android.util.Log import com.itextpdf.io.font.PdfEncodings import com.itextpdf.io.image.ImageDataFactory -import com.itextpdf.kernel.colors.Color import com.itextpdf.kernel.colors.ColorConstants import com.itextpdf.kernel.font.PdfFont import com.itextpdf.kernel.font.PdfFontFactory @@ -41,19 +40,10 @@ import com.itextpdf.kernel.pdf.canvas.parser.PdfCanvasProcessor import com.itextpdf.kernel.pdf.canvas.parser.data.IEventData import com.itextpdf.kernel.pdf.canvas.parser.data.ImageRenderInfo import com.itextpdf.kernel.pdf.canvas.parser.listener.IEventListener -import com.itextpdf.kernel.pdf.extgstate.PdfExtGState import com.itextpdf.kernel.pdf.xobject.PdfImageXObject -import com.itextpdf.layout.Canvas import com.itextpdf.layout.Document import com.itextpdf.layout.element.AreaBreak import com.itextpdf.layout.element.Image -import com.itextpdf.layout.element.Paragraph -import com.itextpdf.layout.layout.LayoutArea -import com.itextpdf.layout.layout.LayoutContext -import com.itextpdf.layout.layout.LayoutResult -import com.itextpdf.layout.properties.TextAlignment -import com.itextpdf.layout.properties.VerticalAlignment -import com.itextpdf.layout.renderer.IRenderer import org.json.JSONArray import org.json.JSONException import org.json.JSONObject @@ -323,133 +313,92 @@ class PDFUtils { } } + // OCR text layer geometry. A tesseract line box spans ascenders to descenders: + // the font size is the box height and the baseline sits just above the box bottom. + // Keep in sync with PDFCanvas.ts (ios). + private const val OCR_FONT_SIZE_RATIO = 1f + private const val OCR_BASELINE_RATIO = 0.8f + private const val OCR_MIN_HORIZONTAL_SCALE = 25f + private const val OCR_MAX_HORIZONTAL_SCALE = 400f + /** - * Try decreasing font size until the Paragraph fits inside the box. + * Draw the ocr text layer on top of the image drawn at [posX], [posY] with + * size [drawnWidth] x [drawnHeight] (in pdf points). + * Each block is drawn as a single run placed on its baseline and horizontally + * scaled to cover exactly the detected box: no reflow, no wrapping. */ - fun findFittingFontSize( - text: String, - font: PdfFont, - box: Rectangle, - pageNumber: Int, - layoutDoc: Document, - maxFontSize: Float, - minFontSize: Float = 4f, - step: Float = 0.5f - ): Float { - var size = maxFontSize - while (size >= minFontSize) { - val p = Paragraph(text).setFont(font).setFontSize(size).setMultipliedLeading(1f).setMargin(0f).setFirstLineIndent(0f).setPadding(0f) - .setTextAlignment(TextAlignment.LEFT) - .setVerticalAlignment(VerticalAlignment.TOP) - val renderer: IRenderer = p.createRendererSubTree() - renderer.parent = layoutDoc.renderer // uses LayoutDocument.getRenderer() under the hood - try { - val layoutArea = LayoutArea(pageNumber, box) - val result = renderer.layout(LayoutContext(layoutArea)) - - // LayoutResult.FULL means the whole paragraph fits in the provided area - if (result.status == LayoutResult.FULL) { - return size - } - } catch (e: Exception) { - // Catch and log the exception so you can see why it crashed for this size. - // Don't rethrow here — we try smaller sizes as fallback. - System.err.println("layout() failed at size $size -> ${e::class.simpleName}: ${e.message}") - e.printStackTrace() - } - size -= step - } - return minFontSize - } - - fun drawTextInBox( - pdfCanvas: PdfCanvas, - pdf: PdfDocument, - box: Rectangle, - text: String, - font: PdfFont, - fontSize: Float, - color: Color = ColorConstants.BLACK, - textRenderingMode: Int, - hAlign: TextAlignment = TextAlignment.LEFT, // LEFT, CENTER, RIGHT, JUSTIFIED - vAlign: VerticalAlignment = VerticalAlignment.TOP // TOP, MIDDLE, BOTTOM - ) { - val canvas = Canvas(pdfCanvas, box) - .setTextRenderingMode(textRenderingMode) - .setFont(font) - .setFontSize(fontSize) - .setFontColor(color) - - canvas.showTextAligned( - text, - when (hAlign) { - TextAlignment.LEFT -> box.left - TextAlignment.CENTER -> box.left + box.width / 2 - TextAlignment.RIGHT -> box.right - else -> box.left // fallback for JUSTIFIED etc. - }, - when (vAlign) { - VerticalAlignment.TOP -> box.top - VerticalAlignment.MIDDLE -> box.bottom + box.height / 2 - VerticalAlignment.BOTTOM -> box.bottom - }, - hAlign, - vAlign, - 0f - ) - } - private fun drawOCRData( pdfDoc: PdfDocument, - doc: Document, page: JSONObject, posX: Float, posY: Float, - imageScale: Float, - toDrawHeight: Float, - textScale: Float, + drawnWidth: Float, + drawnHeight: Float, debug: Boolean, fontCache: FontCache ) { - val ocrData = page.optJSONObject("ocrData") - if (ocrData != null) { - // val imageWidth = ocrData.getDouble("imageWidth").toFloat() - val imageHeight = ocrData.getDouble("imageHeight").toFloat() - val blocks = ocrData.getJSONArray("blocks") - // canvas - for (i in 0.. + val lineTop = boxTop - lineIndex * lineHeight + val textWidth = font.getWidth(line, fontSize) + // stretch the run so that it covers exactly the detected box + val horizontalScale = if (textWidth > 0) { + (100f * boxWidth / textWidth).coerceIn(OCR_MIN_HORIZONTAL_SCALE, OCR_MAX_HORIZONTAL_SCALE) + } else { + 100f + } - val gState = PdfExtGState() - gState.fillOpacity = 0.5f - // draw a debug rectangle if (debug) { - canvas.saveState().rectangle(rect) - // .setExtGState(gState) + canvas.saveState() + .rectangle(Rectangle(left, lineTop - lineHeight, boxWidth, lineHeight)) .setFillColor(ColorConstants.WHITE) .fill().restoreState() } - val text = block.getString("text") - val font = fontCache.getFont(text) - val actualFontSize = findFittingFontSize(text, font, rect, pdfDoc.numberOfPages, doc, fontSize, 4.0f, 2.0f) - - drawTextInBox(canvas, pdfDoc, rect, text, font, actualFontSize, if (debug) ColorConstants.RED else ColorConstants.BLACK, if (debug) PdfCanvasConstants.TextRenderingMode.FILL else PdfCanvasConstants.TextRenderingMode.INVISIBLE) + canvas.saveState() + .beginText() + .setTextRenderingMode(if (debug) PdfCanvasConstants.TextRenderingMode.FILL else PdfCanvasConstants.TextRenderingMode.INVISIBLE) + .setFillColor(if (debug) ColorConstants.RED else ColorConstants.BLACK) + .setFontAndSize(font, fontSize) + .setHorizontalScaling(horizontalScale) + .setTextMatrix(left, lineTop - lineHeight * OCR_BASELINE_RATIO) + .showText(line) + .endText() + .restoreState() } } } @@ -480,7 +429,6 @@ class PDFUtils { val overwrite = jsonOps.optBoolean("overwrite", false) val debug = jsonOps.optBoolean("debug", false) val pagePadding = jsonOps.optInt("page_padding", 0).toFloat() - val textScale = jsonOps.optDouble("text_scale", 3.0).toFloat() val imageScale = jsonOps.optDouble("image_page_scale", 2.0).toFloat() // var pagePadding = 100F var itemsPerPage = jsonOps.optInt("items_per_page", 1) @@ -549,7 +497,6 @@ class PDFUtils { colorMatrix, loadImageOptions ) ?: continue - var imageRatio = image.imageHeight / imageHeight val pageSize = if (imageRotation % 180 != 0) PageSize( image.imageHeight, @@ -566,12 +513,12 @@ class PDFUtils { } document.add(image) if (drawOcrText) { + // the image fills the whole page (page sized from it, no margins) drawOCRData( - pdfDoc, document, page, + pdfDoc, page, 0F, 0F, - imageRatio.toFloat(), - imageHeight.toFloat(), - textScale, debug, + pageSize.width, pageSize.height, + debug, fontCache ) } @@ -660,12 +607,10 @@ class PDFUtils { } var reqWidth = toDrawWidth * imageScale var reqHeight = toDrawHeight * imageScale - var imageRatio = toDrawHeight / imageHeight if (imageRotation % 180 != 0) { val temp = reqWidth reqWidth = reqHeight reqHeight = temp - imageRatio = toDrawHeight / imageWidth } val image = loadImage( @@ -685,8 +630,10 @@ class PDFUtils { val posX = ddx + itemAvailableWidth / 2 - toDrawWidth.toFloat() / 2 - var posY = ddy + itemAvailableHeight / 2 - toDrawHeight.toFloat() / 2 - + // where the image actually lands on the page, before the rotation anchor fixups + val imagePosY = ddy + itemAvailableHeight / 2 - toDrawHeight.toFloat() / 2 + var posY = imagePosY + if ((imageRotation % 360) == 180) { posY += toDrawHeight.toFloat() } else if ((imageRotation % 360) == 90) { @@ -696,9 +643,10 @@ class PDFUtils { document!!.add(image) if (drawOcrText) { drawOCRData( - pdfDoc, document, page, - posX, posY, - imageRatio.toFloat(), toDrawHeight.toFloat(), textScale, debug, + pdfDoc, page, + posX, imagePosY, + toDrawWidth.toFloat(), toDrawHeight.toFloat(), + debug, fontCache ) } diff --git a/tools b/tools index b55093ad..01fb9549 160000 --- a/tools +++ b/tools @@ -1 +1 @@ -Subproject commit b55093ad61fffe0b9171dcd7bc2e59df3b813169 +Subproject commit 01fb9549753f821d203e4b011ae13e046ed3140e diff --git a/yarn.lock b/yarn.lock index 0129fff4..d701dc69 100644 --- a/yarn.lock +++ b/yarn.lock @@ -27,7 +27,7 @@ __metadata: "@akylas/nativescript-app-tools@file:tools::locator=root-workspace-0b6124%40workspace%3A.": version: 1.0.0 - resolution: "@akylas/nativescript-app-tools@file:tools#tools::hash=e75cdc&locator=root-workspace-0b6124%40workspace%3A." + resolution: "@akylas/nativescript-app-tools@file:tools#tools::hash=5f9fe8&locator=root-workspace-0b6124%40workspace%3A." dependencies: "@dotenvx/dotenvx": "npm:1.51.4" "@nativescript-community/fontmin": "npm:0.9.11" @@ -81,7 +81,7 @@ __metadata: typescript: "npm:5.9.3" typescript-eslint: "npm:8.53.0" webpack-bundle-analyzer: "npm:4.10.2" - checksum: 10/11d05f03cc963b603760589644e80a1a723366da39b3b39b08f68f0ffe3868fed6db0c35d8c3b084280193e28ce804c1247bdc884f282fc6b8e73e9f18d17690 + checksum: 10/30d1754f73fe8af03a96fde42c1e35bf7fbcf4a33dc77c6b01589fc45ef18bf4f10282291837a75d3ca4bc8c55c4c0f8ce062c05600baff4851caedbeff6e584 languageName: node linkType: hard