Skip to content

Commit 4a65796

Browse files
tballisontballison
authored
1 parent 7f69cc6 commit 4a65796

17 files changed

Lines changed: 2884 additions & 3 deletions

File tree

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,30 @@
125125
</dependencies>
126126
<build>
127127
<plugins>
128+
<plugin>
129+
<groupId>de.jflex</groupId>
130+
<artifactId>jflex-maven-plugin</artifactId>
131+
<version>1.9.1</version>
132+
<executions>
133+
<execution>
134+
<goals>
135+
<goal>generate</goal>
136+
</goals>
137+
<configuration>
138+
<lexDefinitions>
139+
<lexDefinition>src/main/jflex</lexDefinition>
140+
</lexDefinitions>
141+
</configuration>
142+
</execution>
143+
</executions>
144+
</plugin>
145+
<plugin>
146+
<groupId>org.apache.maven.plugins</groupId>
147+
<artifactId>maven-checkstyle-plugin</artifactId>
148+
<configuration>
149+
<excludes>**/rtf/jflex/RTFTokenizer.java</excludes>
150+
</configuration>
151+
</plugin>
128152
<plugin>
129153
<groupId>org.apache.rat</groupId>
130154
<artifactId>apache-rat-plugin</artifactId>

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,13 @@ public class OfficeParserConfig implements Serializable {
3939

4040
private boolean writeSelectHeadersInBody = false;
4141

42+
/**
43+
* Maximum bytes per embedded object/pict when extracting from RTF within
44+
* MSG files. Since embedded data is streamed to disk (not held in memory),
45+
* the default is 2 GB. Set to -1 for unlimited.
46+
*/
47+
private int rtfEmbeddedMaxBytesInKb = 2 * 1024 * 1024; // 2 GB
48+
4249
private boolean includeGlossary = true;
4350
private String dateOverrideFormat = null;
4451
private int maxOverride = 0;//ignore
@@ -319,4 +326,17 @@ public boolean isWriteSelectHeadersInBody() {
319326
public void setWriteSelectHeadersInBody(boolean writeSelectHeadersInBody) {
320327
this.writeSelectHeadersInBody = writeSelectHeadersInBody;
321328
}
329+
330+
/**
331+
* Maximum bytes (in KB) per embedded object/pict when extracting from RTF
332+
* within MSG files. Data is streamed to disk, so the default is 2 GB.
333+
* Set to -1 for unlimited.
334+
*/
335+
public int getRtfEmbeddedMaxBytesInKb() {
336+
return rtfEmbeddedMaxBytesInKb;
337+
}
338+
339+
public void setRtfEmbeddedMaxBytesInKb(int rtfEmbeddedMaxBytesInKb) {
340+
this.rtfEmbeddedMaxBytesInKb = rtfEmbeddedMaxBytesInKb;
341+
}
322342
}

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,8 @@
8484
import org.apache.tika.parser.html.JSoupParser;
8585
import org.apache.tika.parser.mailcommons.MailDateParser;
8686
import org.apache.tika.parser.microsoft.msg.ExtendedMetadataExtractor;
87-
import org.apache.tika.parser.microsoft.msg.RTFEncapsulatedHTMLExtractor;
8887
import org.apache.tika.parser.microsoft.rtf.RTFParser;
88+
import org.apache.tika.parser.microsoft.rtf.jflex.RTFHtmlDecapsulator;
8989
import org.apache.tika.parser.txt.CharsetDetector;
9090
import org.apache.tika.parser.txt.CharsetMatch;
9191
import org.apache.tika.sax.BodyContentHandler;
@@ -600,8 +600,11 @@ private void handleBodyChunks(Chunk htmlChunk, Chunk rtfChunk, Chunk textChunk,
600600
new MAPIRtfAttribute(MAPIProperty.RTF_COMPRESSED, Types.BINARY.getId(),
601601
chunk.getValue());
602602
byte[] rtfData = rtf.getData();
603-
// Try to extract encapsulated HTML — returns null if not present
604-
String html = RTFEncapsulatedHTMLExtractor.extract(rtfData);
603+
// Try to extract encapsulated HTML + embedded objects in one pass
604+
RTFHtmlDecapsulator decapsulator =
605+
new RTFHtmlDecapsulator(xhtml, parseContext,
606+
officeParserConfig.getRtfEmbeddedMaxBytesInKb());
607+
String html = decapsulator.extract(rtfData);
605608
if (html != null) {
606609
parseHtmlString(html, xhtml, contentIdNames);
607610
parentMetadata.add(MAPI.BODY_TYPES_PROCESSED,

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/jflex/RTFCharsetMaps.java‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.tika.parser.microsoft.rtf.jflex;
18+
19+
import java.nio.charset.Charset;
20+
import java.nio.charset.StandardCharsets;
21+
import java.util.Collections;
22+
import java.util.HashMap;
23+
import java.util.Map;
24+
25+
import org.apache.tika.utils.CharsetUtils;
26+
27+
/**
28+
* Shared charset maps for RTF parsing. Maps RTF {@code \fcharsetN} and
29+
* {@code \ansicpgN} values to Java {@link Charset} instances.
30+
*
31+
* <p>Extracted from the original {@code TextExtractor} so both the JFlex-based
32+
* parser and decapsulator can reuse them.</p>
33+
*/
34+
public final class RTFCharsetMaps {
35+
36+
public static final Charset WINDOWS_1252 = Charset.forName("windows-1252");
37+
38+
/**
39+
* Maps {@code \fcharsetN} values to Java charsets.
40+
* The RTF font table uses these to declare per-font character encodings.
41+
*/
42+
public static final Map<Integer, Charset> FCHARSET_MAP;
43+
44+
/**
45+
* Maps {@code \ansicpgN} values to Java charsets.
46+
* This is the global ANSI code page declared in the RTF header.
47+
*/
48+
public static final Map<Integer, Charset> ANSICPG_MAP;
49+
50+
static {
51+
Map<Integer, Charset> fcharset = new HashMap<>();
52+
53+
fcharset.put(0, WINDOWS_1252); // ANSI
54+
// charset 1 = Default, charset 2 = Symbol
55+
56+
fcharset.put(77, getCharset("MacRoman")); // Mac Roman
57+
fcharset.put(78, getCharset("Shift_JIS")); // Mac Shift Jis
58+
fcharset.put(79, getCharset("ms949")); // Mac Hangul
59+
fcharset.put(80, getCharset("GB2312")); // Mac GB2312
60+
fcharset.put(81, getCharset("Big5")); // Mac Big5
61+
fcharset.put(82, getCharset("johab")); // Mac Johab (old)
62+
fcharset.put(83, getCharset("MacHebrew")); // Mac Hebrew
63+
fcharset.put(84, getCharset("MacArabic")); // Mac Arabic
64+
fcharset.put(85, getCharset("MacGreek")); // Mac Greek
65+
fcharset.put(86, getCharset("MacTurkish")); // Mac Turkish
66+
fcharset.put(87, getCharset("MacThai")); // Mac Thai
67+
fcharset.put(88, getCharset("cp1250")); // Mac East Europe
68+
fcharset.put(89, getCharset("cp1251")); // Mac Russian
69+
70+
fcharset.put(128, getCharset("MS932")); // Shift JIS
71+
fcharset.put(129, getCharset("ms949")); // Hangul
72+
fcharset.put(130, getCharset("ms1361")); // Johab
73+
fcharset.put(134, getCharset("ms936")); // GB2312
74+
fcharset.put(136, getCharset("ms950")); // Big5
75+
fcharset.put(161, getCharset("cp1253")); // Greek
76+
fcharset.put(162, getCharset("cp1254")); // Turkish
77+
fcharset.put(163, getCharset("cp1258")); // Vietnamese
78+
fcharset.put(177, getCharset("cp1255")); // Hebrew
79+
fcharset.put(178, getCharset("cp1256")); // Arabic
80+
fcharset.put(186, getCharset("cp1257")); // Baltic
81+
82+
fcharset.put(204, getCharset("cp1251")); // Russian
83+
fcharset.put(222, getCharset("ms874")); // Thai
84+
fcharset.put(238, getCharset("cp1250")); // Eastern European
85+
fcharset.put(254, getCharset("cp437")); // PC 437
86+
fcharset.put(255, getCharset("cp850")); // OEM
87+
88+
FCHARSET_MAP = Collections.unmodifiableMap(fcharset);
89+
}
90+
91+
static {
92+
Map<Integer, Charset> ansicpg = new HashMap<>();
93+
94+
ansicpg.put(437, getCharset("CP437")); // US IBM
95+
ansicpg.put(708, getCharset("ISO-8859-6")); // Arabic (ASMO 708)
96+
ansicpg.put(709, getCharset("windows-709")); // Arabic (ASMO 449+)
97+
ansicpg.put(710, getCharset("windows-710")); // Arabic (transparent)
98+
ansicpg.put(711, getCharset("windows-711")); // Arabic (Nafitha)
99+
ansicpg.put(720, getCharset("windows-720")); // Arabic (transparent ASMO)
100+
ansicpg.put(819, getCharset("CP819")); // Windows 3.1 (US/Western)
101+
ansicpg.put(850, getCharset("CP850")); // IBM Multilingual
102+
ansicpg.put(852, getCharset("CP852")); // Eastern European
103+
ansicpg.put(860, getCharset("CP860")); // Portuguese
104+
ansicpg.put(862, getCharset("CP862")); // Hebrew
105+
ansicpg.put(863, getCharset("CP863")); // French Canadian
106+
ansicpg.put(864, getCharset("CP864")); // Arabic
107+
ansicpg.put(865, getCharset("CP865")); // Norwegian
108+
ansicpg.put(866, getCharset("CP866")); // Soviet Union
109+
ansicpg.put(874, getCharset("MS874")); // Thai
110+
ansicpg.put(932, getCharset("MS932")); // Japanese
111+
ansicpg.put(936, getCharset("MS936")); // Simplified Chinese
112+
ansicpg.put(949, getCharset("CP949")); // Korean
113+
ansicpg.put(950, getCharset("CP950")); // Traditional Chinese
114+
ansicpg.put(1250, getCharset("CP1250")); // Eastern European
115+
ansicpg.put(1251, getCharset("CP1251")); // Cyrillic
116+
ansicpg.put(1252, getCharset("CP1252")); // Western European
117+
ansicpg.put(1253, getCharset("CP1253")); // Greek
118+
ansicpg.put(1254, getCharset("CP1254")); // Turkish
119+
ansicpg.put(1255, getCharset("CP1255")); // Hebrew
120+
ansicpg.put(1256, getCharset("CP1256")); // Arabic
121+
ansicpg.put(1257, getCharset("CP1257")); // Baltic
122+
ansicpg.put(1258, getCharset("CP1258")); // Vietnamese
123+
ansicpg.put(1361, getCharset("x-Johab")); // Johab
124+
ansicpg.put(10000, getCharset("MacRoman")); // Mac Roman
125+
ansicpg.put(10001, getCharset("Shift_JIS")); // Mac Japan
126+
ansicpg.put(10004, getCharset("MacArabic")); // Mac Arabic
127+
ansicpg.put(10005, getCharset("MacHebrew")); // Mac Hebrew
128+
ansicpg.put(10006, getCharset("MacGreek")); // Mac Greek
129+
ansicpg.put(10007, getCharset("MacCyrillic")); // Mac Cyrillic
130+
ansicpg.put(10029, getCharset("x-MacCentralEurope")); // Mac Latin2
131+
ansicpg.put(10081, getCharset("MacTurkish")); // Mac Turkish
132+
ansicpg.put(57002, getCharset("x-ISCII91")); // Devanagari
133+
ansicpg.put(57003, getCharset("windows-57003")); // Bengali
134+
ansicpg.put(57004, getCharset("windows-57004")); // Tamil
135+
ansicpg.put(57005, getCharset("windows-57005")); // Telugu
136+
ansicpg.put(57006, getCharset("windows-57006")); // Assamese
137+
ansicpg.put(57007, getCharset("windows-57007")); // Oriya
138+
ansicpg.put(57008, getCharset("windows-57008")); // Kannada
139+
ansicpg.put(57009, getCharset("windows-57009")); // Malayalam
140+
ansicpg.put(57010, getCharset("windows-57010")); // Gujarati
141+
ansicpg.put(57011, getCharset("windows-57011")); // Punjabi
142+
143+
ANSICPG_MAP = Collections.unmodifiableMap(ansicpg);
144+
}
145+
146+
private RTFCharsetMaps() {
147+
}
148+
149+
/**
150+
* Resolve a charset by name, falling back to US-ASCII if unavailable.
151+
*/
152+
static Charset getCharset(String name) {
153+
try {
154+
return CharsetUtils.forName(name);
155+
} catch (IllegalArgumentException e) {
156+
return StandardCharsets.US_ASCII;
157+
}
158+
}
159+
160+
/**
161+
* Resolve an ANSI code page number to a Java Charset.
162+
* Tries the ANSICPG_MAP first, then falls back to {@code windows-N} and {@code cpN}.
163+
* Returns {@code WINDOWS_1252} if nothing matches.
164+
*/
165+
public static Charset resolveCodePage(int cpNumber) {
166+
Charset cs = ANSICPG_MAP.get(cpNumber);
167+
if (cs != null) {
168+
return cs;
169+
}
170+
try {
171+
return Charset.forName("windows-" + cpNumber);
172+
} catch (Exception e) {
173+
try {
174+
return Charset.forName("cp" + cpNumber);
175+
} catch (Exception e2) {
176+
return WINDOWS_1252;
177+
}
178+
}
179+
}
180+
}

0 commit comments

Comments
 (0)