Skip to content

Commit aca20dc

Browse files
krickertkrickert
authored
TIKA-4770: Add a Markdown parser with structured, lossless XHTML output (#2922)
1 parent 88220e1 commit aca20dc

15 files changed

Lines changed: 765 additions & 13 deletions

File tree

‎tika-bom/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,11 @@
255255
<version>${revision}</version>
256256
</dependency>
257257

258+
<dependency>
259+
<groupId>org.apache.tika</groupId>
260+
<artifactId>tika-parser-datauri-commons</artifactId>
261+
<version>${revision}</version>
262+
</dependency>
258263
<dependency>
259264
<groupId>org.apache.tika</groupId>
260265
<artifactId>tika-parser-digest-commons</artifactId>

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ public void testNonDetectingDetectorParams() throws Exception {
147147
List<Parser> parsers = new ArrayList<>();
148148
findEncodingDetectionParsers(p, parsers);
149149

150-
assertEquals(7, parsers.size());
150+
assertEquals(8, parsers.size());
151151
EncodingDetector encodingDetector =
152152
((AbstractEncodingDetectorParser) parsers.get(0)).getEncodingDetector();
153153
assertTrue(encodingDetector instanceof CompositeEncodingDetector);
@@ -177,7 +177,7 @@ public void testConfigurabilityOfUserSpecified() throws Exception {
177177
List<Parser> parsers = new ArrayList<>();
178178
findEncodingDetectionParsers(p, parsers);
179179

180-
assertEquals(8, parsers.size());
180+
assertEquals(9, parsers.size());
181181

182182
for (Parser encodingDetectingParser : parsers) {
183183
EncodingDetector encodingDetector =
@@ -203,7 +203,7 @@ public void testMarkLimitUnit() throws Exception {
203203
List<Parser> parsers = new ArrayList<>();
204204
findEncodingDetectionParsers(p, parsers);
205205

206-
assertEquals(7, parsers.size());
206+
assertEquals(8, parsers.size());
207207
for (Parser childParser : parsers) {
208208
EncodingDetector encodingDetector =
209209
((AbstractEncodingDetectorParser) childParser).getEncodingDetector();

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
</dependencies>
6262
<modules>
6363
<module>tika-parser-jdbc-commons</module>
64+
<module>tika-parser-datauri-commons</module>
6465
<module>tika-parser-digest-commons</module>
6566
<module>tika-parser-mail-commons</module>
6667
<module>tika-parser-xmp-commons</module>

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/pom.xml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
-->
20+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<parent>
22+
<artifactId>tika-parsers-standard-modules</artifactId>
23+
<groupId>org.apache.tika</groupId>
24+
<version>${revision}</version>
25+
<relativePath>../pom.xml</relativePath>
26+
</parent>
27+
<modelVersion>4.0.0</modelVersion>
28+
29+
<artifactId>tika-parser-datauri-commons</artifactId>
30+
<name>Apache Tika data URI commons</name>
31+
32+
<dependencies>
33+
<dependency>
34+
<groupId>commons-codec</groupId>
35+
<artifactId>commons-codec</artifactId>
36+
</dependency>
37+
<dependency>
38+
<groupId>commons-io</groupId>
39+
<artifactId>commons-io</artifactId>
40+
</dependency>
41+
</dependencies>
42+
43+
<build>
44+
<plugins>
45+
<plugin>
46+
<groupId>org.apache.maven.plugins</groupId>
47+
<artifactId>maven-jar-plugin</artifactId>
48+
<configuration>
49+
<archive>
50+
<manifestEntries>
51+
<Automatic-Module-Name>org.apache.tika.parser.datauri.commons</Automatic-Module-Name>
52+
</manifestEntries>
53+
</archive>
54+
</configuration>
55+
</plugin>
56+
</plugins>
57+
</build>
58+
</project>

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DataURIScheme.java renamed to tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/main/java/org/apache/tika/parser/datauri/DataURIScheme.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
package org.apache.tika.parser.html;
17+
package org.apache.tika.parser.datauri;
1818

1919
import java.io.InputStream;
2020
import java.util.Arrays;

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DataURISchemeParseException.java renamed to tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/main/java/org/apache/tika/parser/datauri/DataURISchemeParseException.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
package org.apache.tika.parser.html;
17+
package org.apache.tika.parser.datauri;
1818

1919
import org.apache.tika.exception.TikaException;
2020

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DataURISchemeUtil.java renamed to tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/main/java/org/apache/tika/parser/datauri/DataURISchemeUtil.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
package org.apache.tika.parser.html;
17+
package org.apache.tika.parser.datauri;
1818

1919
import java.nio.charset.Charset;
2020
import java.nio.charset.IllegalCharsetNameException;
@@ -96,7 +96,6 @@ public List<DataURIScheme> extract(String string) {
9696
}
9797
list.add(dataURIScheme);
9898
}
99-
return (list == null) ? Collections.EMPTY_LIST : list;
99+
return (list == null) ? Collections.emptyList() : list;
100100
}
101-
102101
}

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/DataURISchemeParserTest.java renamed to tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-datauri-commons/src/test/java/org/apache/tika/parser/datauri/DataURISchemeParserTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
package org.apache.tika.parser.html;
17+
package org.apache.tika.parser.datauri;
1818

1919
import static org.junit.jupiter.api.Assertions.assertEquals;
2020
import static org.junit.jupiter.api.Assertions.assertFalse;

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,9 @@
4545
</dependency>
4646

4747
<dependency>
48-
<groupId>commons-codec</groupId>
49-
<artifactId>commons-codec</artifactId>
48+
<groupId>org.apache.tika</groupId>
49+
<artifactId>tika-parser-datauri-commons</artifactId>
50+
<version>${project.version}</version>
5051
</dependency>
5152

5253
<!-- test scope -->

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/HtmlHandler.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@
4545
import org.apache.tika.metadata.TikaCoreProperties;
4646
import org.apache.tika.mime.MediaType;
4747
import org.apache.tika.parser.ParseContext;
48+
import org.apache.tika.parser.datauri.DataURIScheme;
49+
import org.apache.tika.parser.datauri.DataURISchemeParseException;
50+
import org.apache.tika.parser.datauri.DataURISchemeUtil;
4851
import org.apache.tika.sax.TextContentHandler;
4952
import org.apache.tika.sax.XHTMLContentHandler;
5053
import org.apache.tika.utils.StringUtils;

0 commit comments

Comments
 (0)