Skip to content

Commit 4e8d528

Browse files
vitali-prvitali-pr
committed
Improve sorting algorithms for rotated OCR output
Split lines if words are not close to each other. DEVSIX-9901
1 parent 6903c00 commit 4e8d528

31 files changed

Lines changed: 195 additions & 76 deletions

‎pdfocr-api/src/main/java/com/itextpdf/pdfocr/util/PdfOcrTextBuilder.java‎

Lines changed: 97 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@ This file is part of the iText (R) project.
3838
* Class to build text output from the provided image OCR result and write it to the TXT file.
3939
*/
4040
public final class PdfOcrTextBuilder {
41-
private static final float DEFAULT_INTERSECTION_THRESHOLD = 0.55F;
41+
private static final double DEFAULT_INTERSECTION_THRESHOLD = 0.55;
42+
private static final double DEFAULT_DISTANCE_THRESHOLD = 2;
4243
private static final double DEFAULT_ANGLE_THRESHOLD = Math.toRadians(10);
4344
private static final double EPS = 1e-6;
4445

@@ -151,35 +152,77 @@ public static void collectWordsIntoLines(Map<Integer, List<TextInfo>> textInfos)
151152
* element contains a word or a line and its 4 coordinates (bbox)
152153
*/
153154
public static void sortTextInfosByLines(Map<Integer, List<TextInfo>> textInfos) {
154-
for (Map.Entry<Integer, List<TextInfo>> entry : textInfos.entrySet()) {
155-
Collections.sort(entry.getValue(), new Comparator<TextInfo>() {
156-
@Override
157-
public int compare(TextInfo first, TextInfo second) {
158-
// Not really needed, but just in case.
159-
if (first == second) {
160-
return 0;
161-
}
155+
List<Integer> pages = textInfos.keySet().stream().sorted().collect(Collectors.toList());
156+
for (int pageNr : pages) {
157+
List<TextInfo> originals = textInfos.get(pageNr);
158+
if (originals == null || originals.size() <= 1) {
159+
continue;
160+
}
162161

163-
double angleDiff = getAngleDiff(first, second);
164-
if (Math.abs(angleDiff) > DEFAULT_ANGLE_THRESHOLD) {
165-
double firstRoundAngle = roundAngle(first.getRotationAngle(), DEFAULT_ANGLE_THRESHOLD);
166-
double secondRoundAngle = roundAngle(second.getRotationAngle(), DEFAULT_ANGLE_THRESHOLD);
167-
return Double.compare(firstRoundAngle, secondRoundAngle);
162+
// Group by rotation: group TextInfo items whose angles are within DEFAULT_ANGLE_THRESHOLD
163+
List<List<TextInfo>> rotGroups = new ArrayList<>();
164+
List<Double> rotations = new ArrayList<>();
165+
166+
for (TextInfo ti : originals) {
167+
final double angle = ti.getRotationAngle();
168+
int toPlace = -1;
169+
double bestDiff = DEFAULT_ANGLE_THRESHOLD;
170+
for (int i = 0; i < rotations.size(); ++i) {
171+
final double angleDiff = Math.abs(getAngleDiff(ti, rotGroups.get(i).get(0)));
172+
if (angleDiff <= bestDiff) {
173+
bestDiff = angleDiff;
174+
toPlace = i;
168175
}
176+
}
177+
if (toPlace == -1) {
178+
List<TextInfo> g = new ArrayList<>();
179+
g.add(ti);
180+
rotGroups.add(g);
181+
rotations.add(angle);
182+
} else {
183+
rotGroups.get(toPlace).add(ti);
184+
}
185+
}
186+
187+
// Sort by angle
188+
List<Integer> indices = new ArrayList<>(rotations.size());
189+
for (int i = 0; i < rotations.size(); ++i) {
190+
indices.add(i);
191+
}
192+
Collections.sort(indices, (first, second) -> {
193+
return Double.compare(roundAngle(rotations.get(first), DEFAULT_ANGLE_THRESHOLD),
194+
roundAngle(rotations.get(second), DEFAULT_ANGLE_THRESHOLD));
195+
});
196+
197+
List<TextInfo> result = new ArrayList<>(originals.size());
198+
for (int idx : indices) {
199+
List<TextInfo> group = rotGroups.get(idx);
200+
Collections.sort(group, new Comparator<TextInfo>() {
201+
@Override
202+
public int compare(TextInfo first, TextInfo second) {
203+
// Not really needed, but just in case
204+
if (first == second) {
205+
return 0;
206+
}
169207

170-
BoundingBox[] boxes = BoundingBox.getNormalizedBBoxes(first, second);
171-
BoundingBox box1 = boxes[0];
172-
BoundingBox box2 = boxes[1];
208+
BoundingBox[] boxes = BoundingBox.getNormalizedBBoxes(first, second);
209+
BoundingBox box1 = boxes[0];
210+
BoundingBox box2 = boxes[1];
173211

174-
if (!areIntersect(box1, box2)) {
175-
double middleDistPerpendicularDiff =
176-
(box2.minY + box2.getHeight() / 2) - (box1.minY + box1.getHeight() / 2);
177-
return middleDistPerpendicularDiff > 0 ? 1 : -1;
212+
if (!areIntersectOnVertical(box1, box2)) {
213+
double middleDistPerpendicularDiff =
214+
(box2.minY + box2.getHeight() / 2) - (box1.minY + box1.getHeight() / 2);
215+
return middleDistPerpendicularDiff > 0 ? 1 : -1;
216+
}
217+
218+
return Double.compare(box1.minX, box2.minX) > 0 ? 1 : -1;
178219
}
220+
});
179221

180-
return Double.compare(box1.minX, box2.minX) > 0 ? 1 : -1;
181-
}
182-
});
222+
result.addAll(group);
223+
}
224+
225+
textInfos.put(pageNr, result);
183226
}
184227
}
185228

@@ -232,7 +275,7 @@ static boolean isInTheSameLine(TextInfo currentTextInfo, TextInfo previousTextIn
232275
}
233276

234277
BoundingBox[] boxes = BoundingBox.getNormalizedBBoxes(currentTextInfo, previousTextInfo);
235-
return areIntersect(boxes[0], boxes[1]);
278+
return areIntersectOnVertical(boxes[0], boxes[1]) && areCloseOnHorizontal(boxes[0], boxes[1]);
236279
}
237280

238281
/**
@@ -303,23 +346,47 @@ private static void updateBBoxes(List<TextInfo> line) {
303346
}
304347

305348
/**
306-
* Checks whether 2 text chunks are in the same line by their bounding boxes. The horizontal intersection
307-
* determined by the projection onto the y-axis must be more than {@link #DEFAULT_INTERSECTION_THRESHOLD}
308-
* for at least one of the text chunks.
349+
* Checks whether 2 text chunks are in the same line by their bounding boxes.
350+
*
351+
* <p>
352+
* The intersection on vertical is determined by the projection onto the y-axis must be more
353+
* than {@link #DEFAULT_INTERSECTION_THRESHOLD} for at least one of the text chunks.
309354
*
310355
* @param box1 bounding box of the first text chunk
311356
* @param box2 bounding box of the second text chunk
312357
*
313-
* @return {@code true} if chunks intersect horizontally, {@code false} otherwise
358+
* @return {@code true} if chunks intersect on vertical, {@code false} otherwise
314359
*/
315-
private static boolean areIntersect(BoundingBox box1, BoundingBox box2) {
360+
private static boolean areIntersectOnVertical(BoundingBox box1, BoundingBox box2) {
316361
double intersection = Math.min(box1.maxY, box2.maxY) - Math.max(box1.minY, box2.minY);
317362
double firstIntersectPercentage = intersection / box1.getHeight();
318363
double secondIntersectPercentage = intersection / box2.getHeight();
319364

320365
return Math.max(firstIntersectPercentage, secondIntersectPercentage) > DEFAULT_INTERSECTION_THRESHOLD;
321366
}
322367

368+
/**
369+
* Checks whether 2 text chunks are close to each other on the horizontal.
370+
*
371+
* <p>
372+
* If the distance between them is {@link #DEFAULT_DISTANCE_THRESHOLD} times bigger
373+
* than the biggest chunk, such chunks are treated as not close.
374+
*
375+
* @param box1 bounding box of the first text chunk
376+
* @param box2 bounding box of the second text chunk
377+
*
378+
* @return {@code true} if chunks intersect, {@code false} otherwise
379+
*/
380+
private static boolean areCloseOnHorizontal(BoundingBox box1, BoundingBox box2) {
381+
double distance = Math.min(box1.maxX, box2.maxX) - Math.max(box1.minX, box2.minX);
382+
if (distance >= 0) {
383+
// Even intersected
384+
return true;
385+
}
386+
387+
return Math.abs(distance) < Math.max(box1.getWidth(), box2.getWidth()) * DEFAULT_DISTANCE_THRESHOLD;
388+
}
389+
323390
/**
324391
* Merges list of text infos into single line.
325392
*

‎pdfocr-api/src/test/java/com/itextpdf/pdfocr/util/PdfOcrTextBuilderTest.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,20 @@ public void buildTextTest() {
6262
Assertions.assertEquals(expectedResult, actualResult);
6363
}
6464

65+
@Test
66+
public void buildTextDistancedTest() {
67+
Map<Integer, List<TextInfo>> textInfoMap = new HashMap<>();
68+
List<TextInfo> textInfos = new ArrayList<>();
69+
textInfos.add(new TextInfo("Third", new Rectangle(200, 0, 100, 100)));
70+
textInfos.add(new TextInfo("Fourth", new Rectangle(610, 0, 100, 100)));
71+
textInfos.add(new TextInfo("Second", new Rectangle(100, 100, 120, 65)));
72+
textInfos.add(new TextInfo("First", new Rectangle(0, 200, 100, 30)));
73+
textInfoMap.put(1, textInfos);
74+
String actualResult = PdfOcrTextBuilder.buildText(textInfoMap);
75+
String expectedResult = "First\nSecond\nThird\nFourth\n";
76+
Assertions.assertEquals(expectedResult, actualResult);
77+
}
78+
6579
@Test
6680
public void generifyLineTest() {
6781
Map<Integer, List<TextInfo>> textInfoMap = new HashMap<>();
@@ -72,6 +86,7 @@ public void generifyLineTest() {
7286
textInfos.add(new TextInfo("First", new Rectangle(0, 0, 100, 30)));
7387
textInfoMap.put(1, textInfos);
7488
PdfOcrTextBuilder.generifyWordBBoxesByLine(textInfoMap);
89+
textInfos = textInfoMap.get(1);
7590
Assertions.assertTrue(new Rectangle(0, 0, 100, 50).equalsWithEpsilon(textInfos.get(0).getBBoxRect()));
7691
Assertions.assertTrue(new Rectangle(100, 0, 120, 50).equalsWithEpsilon(textInfos.get(1).getBBoxRect()));
7792
Assertions.assertTrue(new Rectangle(200, 0, 100, 50).equalsWithEpsilon(textInfos.get(2).getBBoxRect()));

‎pdfocr-onnx-abstract/src/test/java/com/itextpdf/pdfocr/onnx/OnnxDoImageOcrLanguagesTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ public void chineseDoImageOcrTest() {
108108
File imageFile = new File(src);
109109

110110
String textFromImage = OnnxTestUtils.getTextFromImage(imageFile, OCR_ENGINE);
111-
Assertions.assertEquals("I\n4\n\n-\nnI\nK/i\nhao\n", textFromImage);
111+
Assertions.assertEquals("I\n4\n\n-\nnI\nhao\nK/i\n", textFromImage);
112112
}
113113

114114
@Test
@@ -186,7 +186,7 @@ public void japaneseDoImageOcrTest() {
186186
File imageFile = new File(src);
187187

188188
String textFromImage = OnnxTestUtils.getTextFromImage(imageFile, OCR_ENGINE);
189-
Assertions.assertEquals("B\n*\n-\na\naa\nK\n*\n-\n-\n", textFromImage);
189+
Assertions.assertEquals("B\n*\n-\naa\n-\na\nK\n*\n-\n", textFromImage);
190190
}
191191

192192
@Test

‎pdfocr-onnx-abstract/src/test/java/com/itextpdf/pdfocr/onnx/OnnxIntegrationTest.java‎

Lines changed: 62 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -148,28 +148,32 @@ public void scannedTest() throws IOException {
148148
try (PdfDocument pdfDocument = new PdfDocument(new PdfReader(dest))) {
149149
ExtractionStrategy extractionStrategy = OnnxTestUtils.extractTextFromLayer(pdfDocument, 1, "Text1");
150150
Assertions.assertEquals(DeviceCmyk.MAGENTA, extractionStrategy.getFillColor());
151-
Assertions.assertEquals("la\n" +
152-
"se No\n" +
153-
"-\n" +
154-
"AY SI ENSAYARA COMO ACTUAR?\n" +
155-
"Tanto peor, lo mejor es descansar y no\n" +
156-
"fiesta, si pensar\n" +
157-
"puede. hay nada mas desalentador\n" +
158-
"ver en las fiestas a jovenes con cara de lastima y\n" +
159-
"el\n" +
160-
"iluslonadas y se pasado todo\n" +
161-
"que han dia tratando\n" +
162-
"hallar lo mejor y la mas atractiva manera de pres\n" +
163-
"tarse en publico. Hay que actuar con calma y no\n" +
164-
"cansaremos de repetirlo, Lo mas importante es saber\n" +
165-
"que se va a poner y tener todo a mano,\n" +
166-
"Si intenta probar un nuevo lapiz labial para la a\n" +
167-
"sion, asegurese que armonice con vestido lle\n" +
168-
"-\n" +
169-
"También el el\n" +
170-
"rà. que\n" +
171-
"maquillaje de los ojos debe armoni\n" +
172-
"con el conjunto.", extractionStrategy.getResultantText());
151+
Assertions.assertEquals("la\n"
152+
+ "se\n"
153+
+ "No\n"
154+
+ "-\n"
155+
+ "AY SI ENSAYARA COMO ACTUAR?\n"
156+
+ "Tanto peor, lo mejor es descansar y no\n"
157+
+ "pensar\n"
158+
+ "fiesta, si\n"
159+
+ "puede. hay nada mas desalentador\n"
160+
+ "ver en las fiestas a jovenes con cara de lastima y\n"
161+
+ "iluslonadas y\n"
162+
+ "que han\n"
163+
+ "se pasado todo el\n"
164+
+ "dia tratando\n"
165+
+ "hallar lo mejor y la mas atractiva manera de pres\n"
166+
+ "tarse en publico. Hay que actuar con calma y no\n"
167+
+ "cansaremos de repetirlo, Lo mas importante es saber\n"
168+
+ "que se va a poner y tener todo a mano,\n"
169+
+ "Si intenta probar un nuevo lapiz labial para la a\n"
170+
+ "sion, asegurese que armonice con vestido lle\n"
171+
+ "-\n"
172+
+ "que\n"
173+
+ "rà. También el\n"
174+
+ "maquillaje de los ojos debe armoni\n"
175+
+ "con el conjunto.\n"
176+
+ "el", extractionStrategy.getResultantText());
173177
}
174178
}
175179

@@ -183,16 +187,42 @@ public void halftoneTest() throws IOException {
183187
try (PdfDocument pdfDocument = new PdfDocument(new PdfReader(dest))) {
184188
ExtractionStrategy extractionStrategy = OnnxTestUtils.extractTextFromLayer(pdfDocument, 1, "Text1");
185189
Assertions.assertEquals(DeviceCmyk.MAGENTA, extractionStrategy.getFillColor());
186-
Assertions.assertEquals("Silliness Enablers INVOICE\nYou dream it we enable it\n" +
187-
"Middle of Nowhere\nPhone +32 9 292 22 22 INVOICE #100\n" +
188-
"Fax +32 9 270 00 00 DATE: 6/30/2020\nTO: SHIP TO:\nAndré André Lemos\n" +
189-
"Le emos\nTycoon Corp. Tycoor Corp\nWonderfulStreet Wonderfu Street\n" +
190-
"Lala Land Lala Land\n+351 911 111 111 +351 911 111 111\n" +
191-
"C AMENTS OR SPFCIAI INSTRUCTIONS\nITEMS MUST BE DELIVERED FULLY ASSEMBLED\n" +
192-
"ON P.O NUMBER REQUISITIONER SHIPPED VIA F.O.B POINT TERMS\nS/ ES RSC\n" +
193-
"3Vi #7394009320 V Vebsite form Al R Delivery Due on receipt\n" +
194-
"QUANTITY DESCRIPTION UNIT PRICE TOTAL\n10 Lasers $3000 $30000\n" +
195-
"2 Band-Aids $1 $2\nSharks $99999 $499995"
190+
Assertions.assertEquals("INVOICE\n"
191+
+ "Silliness Enablers\n"
192+
+ "You dream it we enable it\n"
193+
+ "Middle of Nowhere\n"
194+
+ "INVOICE #100\n"
195+
+ "Phone +32 9 292 22 22\n"
196+
+ "DATE: 6/30/2020\n"
197+
+ "Fax +32 9 270 00 00\n"
198+
+ "TO: SHIP TO:\n"
199+
+ "André Lemos\n"
200+
+ "André Le emos\n"
201+
+ "Tycoor Corp\n"
202+
+ "Tycoon Corp.\n"
203+
+ "Wonderfu Street\n"
204+
+ "WonderfulStreet\n"
205+
+ "Lala Land\n"
206+
+ "Lala Land\n"
207+
+ "+351 911 111 111 +351 911 111 111\n"
208+
+ "C AMENTS OR SPFCIAI INSTRUCTIONS\n"
209+
+ "ITEMS MUST BE DELIVERED FULLY ASSEMBLED\n"
210+
+ "P.O NUMBER REQUISITIONER SHIPPED VIA F.O.B POINT TERMS\n"
211+
+ "S/ RSC ON\n"
212+
+ "ES\n"
213+
+ "3Vi #7394009320 V Vebsite form Al R\n"
214+
+ "Delivery Due on receipt\n"
215+
+ "UNIT PRICE TOTAL\n"
216+
+ "DESCRIPTION\n"
217+
+ "QUANTITY\n"
218+
+ "$3000 $30000\n"
219+
+ "10\n"
220+
+ "Lasers\n"
221+
+ "$1 $2\n"
222+
+ "Band-Aids\n"
223+
+ "2\n"
224+
+ "$99999 $499995\n"
225+
+ "Sharks"
196226
, extractionStrategy.getResultantText());
197227
}
198228
}

‎pdfocr-onnx-abstract/src/test/java/com/itextpdf/pdfocr/onnx/actions/OnnxEventHandlingTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -358,7 +358,7 @@ public void createTxtFileTwoImagesTest() throws IOException {
358358
@Test
359359
public void createTxtFileStreamTest() throws IOException {
360360
try (InputStream input = FileUtil.getInputStreamForFile(TEST_IMAGE_DIRECTORY + "numbers_01.jpg");
361-
FileOutputStream output = new FileOutputStream(TEST_IMAGE_DIRECTORY + "createTxtFileStream.txt")) {
361+
FileOutputStream output = new FileOutputStream(DESTINATION_FOLDER + "createTxtFileStream.txt")) {
362362
OCR_ENGINE.createTxtFile(input, output);
363363
}
364364

‎pdfocr-onnx-abstract/src/test/resources/com/itextpdf/pdfocr/OcrPdfTest/cmp_2pages.pdf‎

-28 Bytes
Binary file not shown.

‎pdfocr-onnx-abstract/src/test/resources/com/itextpdf/pdfocr/OcrPdfTest/cmp_layers.pdf‎

-27 Bytes
Binary file not shown.

‎pdfocr-onnx-abstract/src/test/resources/com/itextpdf/pdfocr/OcrPdfTest/cmp_notRecognizable.pdf‎

1 Byte
Binary file not shown.

‎pdfocr-onnx-abstract/src/test/resources/com/itextpdf/pdfocr/OcrPdfTest/cmp_textWithImages.pdf‎

15 Bytes
Binary file not shown.

‎pdfocr-onnx-abstract/src/test/resources/com/itextpdf/pdfocr/OnnxCreateTxtFileTest/cmp_createTxtFile.txt‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,41 @@
11
This a test
22
message for
33
OCR Scanner
4-
ST
5-
-
64
Test
75
BMPTest
6+
ST
7+
-
88
TEST
99
This textIS sideways
1010
DiagonalTxT
1111
SI ENSAYARA COMO ACTUAR?
1212
Tanto peor, lo mejor es descansar no
13-
fiesta, si pensar
14-
puede. hay nada mas desalentador
15-
se No
13+
fiesta, si
14+
pensar
15+
puede. No
1616
la
17+
hay nada mas desalentador
18+
se
1719
ver en las fiestas jovenes con cara de lastima y
18-
iluslonadas se han pasado todo el
19-
que dia tratando
20+
iluslonadas se pasado todo el
21+
que han
22+
dia tratando
2023
hallar lo mejor la mas atractiva manera de pres
2124
tarse en publico. Hay que actuar con calma y
2225
cansaremos de repetirlo, Lo mas importante es saber
2326
que se va a poner tener todo a
2427
Si intenta probar un nuevo lapiz labial para la a
2528
sion, asegurese que armonice con vestido
26-
rà. que lle
27-
También el el
29+
rà.
30+
que lle
31+
También el
32+
el
2833
maquillaje de los ojos debe armoni
2934
con el conjunto.
3035
C
3136
4
32-
oueur A
37+
oueur
38+
A
3339
ou
3440
K
3541
K
@@ -46,7 +52,8 @@ typesetting remaining essentially unchanged. It was popularised in
4652
the 1960s with the release of Letraset sheets containing Lorem
4753
Ipsum passages, and more recently with desktop publishing
4854
software like Aldus PageMaker including versions of Lorem Ipsum.
49-
pue s!
55+
pue
56+
s!
5057
jwMsdI
5158
a
5259
a

0 commit comments

Comments
 (0)