Skip to content

Commit 56397f1

Browse files
tballisontballison
authored
TIKA-4553 -- rm TikaConfig from tika-app (#2431)
1 parent 9e515ff commit 56397f1

11 files changed

Lines changed: 109 additions & 143 deletions

File tree

‎tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@
6868

6969
import org.apache.tika.Tika;
7070
import org.apache.tika.async.cli.TikaAsyncCLI;
71-
import org.apache.tika.config.TikaConfig;
7271
import org.apache.tika.config.TikaConfigSerializer;
72+
import org.apache.tika.config.loader.TikaLoader;
7373
import org.apache.tika.detect.CompositeDetector;
7474
import org.apache.tika.detect.Detector;
7575
import org.apache.tika.exception.TikaException;
@@ -129,7 +129,7 @@ protected ContentHandler getContentHandler(OutputStream output, Metadata metadat
129129
private ParseContext context;
130130
private Detector detector;
131131
private Parser parser;
132-
private TikaConfig config;
132+
private TikaLoader tikaLoader;
133133
private String configFilePath;
134134
private boolean recursiveJSON = false;
135135
private URI networkURI = null;
@@ -518,9 +518,9 @@ public void process(String arg) throws Exception {
518518

519519
private void dumpConfig(TikaConfigSerializer.Mode mode) throws Exception {
520520
configure();
521-
TikaConfig localConfig = (config == null) ? TikaConfig.getDefaultConfig() : config;
522-
523-
TikaConfigSerializer.serialize(localConfig, mode, new OutputStreamWriter(System.out, UTF_8), UTF_8);
521+
TikaLoader localConfig = (tikaLoader == null) ? TikaLoader.loadDefault() : tikaLoader;
522+
//TODO -- implement mode
523+
System.out.println(localConfig.getConfig().toString());
524524
}
525525

526526
private void convertConfigXmlToJson(String paths) throws Exception {
@@ -553,14 +553,16 @@ private void convertConfigXmlToJson(String paths) throws Exception {
553553
private void handleRecursiveJson(URL url, OutputStream output) throws IOException, SAXException, TikaException {
554554
Metadata metadata = new Metadata();
555555
RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
556-
RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(getContentHandlerFactory(type), -1, config.getMetadataFilter());
556+
RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(getContentHandlerFactory(type), -1,
557+
tikaLoader.loadMetadataFilters());
557558
try (InputStream input = TikaInputStream.get(url, metadata)) {
558559
wrapper.parse(input, handler, metadata, context);
559560
}
560561
JsonMetadataList.setPrettyPrinting(prettyPrint);
561562
try (Writer writer = getOutputWriter(output, encoding)) {
562563
List<Metadata> metadataList = handler.getMetadataList();
563-
metadataList = config.getMetadataFilter().filter(metadataList);
564+
metadataList = tikaLoader
565+
.loadMetadataFilters().filter(metadataList);
564566
JsonMetadataList.toJson(metadataList, writer);
565567
}
566568
}
@@ -710,26 +712,32 @@ private boolean testForBatch(String[] args) {
710712

711713
private void configure() throws TikaException, IOException, SAXException {
712714
if (configFilePath != null) {
713-
config = new TikaConfig(new File(configFilePath));
715+
tikaLoader = TikaLoader.load(Paths.get(configFilePath));
714716
} else {
715717
String warn = "As a convenience, TikaCLI has turned on several non-default features\n" +
716-
"as specified in tika-app/src/main/resources/tika-config-default-single-file.xml.\n" +
718+
"as specified in tika-app/src/main/resources/tika-config-default-single-file.json.\n" +
717719
"See: TIKA-2374, TIKA-4017, TIKA-4354 and TIKA-4472).\n" +
718720
"This is not the default behavior in Tika generally or in tika-server.";
719721
LOG.info(warn);
720-
try (InputStream is = getClass().getResourceAsStream("/tika-config-default-single-file.xml")) {
721-
config = new TikaConfig(is);
722+
Path tempConfig = Files.createTempFile("tika-config-", ".json");
723+
try {
724+
try (InputStream is = getClass().getResourceAsStream("/tika-config-default-single-file.json")) {
725+
Files.copy(is, tempConfig, StandardCopyOption.REPLACE_EXISTING);
726+
}
727+
tikaLoader = TikaLoader.load(tempConfig);
728+
} finally {
729+
Files.deleteIfExists(tempConfig);
722730
}
723731
}
724732
if (networkURI != null) {
725733
parser = new NetworkParser(networkURI);
726734
} else {
727-
parser = new AutoDetectParser(config);
735+
parser = tikaLoader.loadAutoDetectParser();
728736
if (digester != null) {
729737
parser = new DigestingParser(parser, digester, false);
730738
}
731739
}
732-
detector = config.getDetector();
740+
detector = tikaLoader.loadDetectors();
733741
context.set(Parser.class, parser);
734742
context.set(PasswordProvider.class, new SimplePasswordProvider(password));
735743
}
@@ -932,9 +940,9 @@ private void compareFileMagic(String magicDir) throws Exception {
932940
}
933941

934942
// See how those compare to the Tika ones
935-
TikaConfig config = TikaConfig.getDefaultConfig();
936-
MimeTypes mimeTypes = config.getMimeRepository();
937-
MediaTypeRegistry registry = config.getMediaTypeRegistry();
943+
TikaLoader loader = TikaLoader.loadDefault();
944+
MimeTypes mimeTypes = TikaLoader.getMimeTypes();
945+
MediaTypeRegistry registry = loader.getMediaTypeRegistry();
938946
for (String mime : fileMimes) {
939947
try {
940948
final MimeType type = mimeTypes.getRegisteredMimeType(mime);

‎tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java‎

Lines changed: 27 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@
3636
import java.net.MalformedURLException;
3737
import java.net.URL;
3838
import java.nio.file.Files;
39+
import java.nio.file.Path;
40+
import java.nio.file.StandardCopyOption;
3941
import java.util.Arrays;
4042
import java.util.HashMap;
4143
import java.util.List;
@@ -70,19 +72,18 @@
7072
import org.xml.sax.SAXException;
7173
import org.xml.sax.helpers.AttributesImpl;
7274

73-
import org.apache.tika.config.TikaConfig;
75+
import org.apache.tika.config.loader.TikaLoader;
76+
import org.apache.tika.exception.TikaConfigException;
7477
import org.apache.tika.exception.TikaException;
7578
import org.apache.tika.extractor.DocumentSelector;
7679
import org.apache.tika.io.TikaInputStream;
7780
import org.apache.tika.metadata.Metadata;
7881
import org.apache.tika.metadata.TikaCoreProperties;
7982
import org.apache.tika.mime.MediaType;
8083
import org.apache.tika.parser.AutoDetectParser;
81-
import org.apache.tika.parser.DigestingParser;
8284
import org.apache.tika.parser.ParseContext;
8385
import org.apache.tika.parser.Parser;
8486
import org.apache.tika.parser.RecursiveParserWrapper;
85-
import org.apache.tika.parser.digestutils.CommonsDigester;
8687
import org.apache.tika.sax.BasicContentHandlerFactory;
8788
import org.apache.tika.sax.BodyContentHandler;
8889
import org.apache.tika.sax.ContentHandlerDecorator;
@@ -154,9 +155,9 @@ public class TikaGUI extends JFrame implements ActionListener, HyperlinkListener
154155
* File chooser.
155156
*/
156157
private final JFileChooser chooser = new JFileChooser();
157-
private final TikaConfig tikaConfig;
158+
private final TikaLoader tikaConfig;
158159

159-
public TikaGUI(Parser parser, TikaConfig tikaConfig) {
160+
public TikaGUI(Parser parser, TikaLoader tikaConfig) {
160161
super("Apache Tika");
161162
this.tikaConfig = tikaConfig;
162163
setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
@@ -194,21 +195,32 @@ public TikaGUI(Parser parser, TikaConfig tikaConfig) {
194195
* @throws Exception if an error occurs
195196
*/
196197
public static void main(String[] args) throws Exception {
197-
TikaConfig config = null;
198+
TikaLoader config = null;
198199
if (args.length > 0) {
199200
File configFile = new File(args[0]);
200-
config = new TikaConfig(configFile);
201+
config = TikaLoader.load(configFile.toPath());
201202
} else {
202-
try (InputStream is = TikaGUI.class.getResourceAsStream("/tika-config-default-single-file.xml")) {
203-
config = new TikaConfig(is);
203+
Path tempConfig = Files.createTempFile("tika-config-", ".json");
204+
try {
205+
try (InputStream is = TikaGUI.class.getResourceAsStream("/tika-config-default-single-file.json")) {
206+
Files.copy(is, tempConfig, StandardCopyOption.REPLACE_EXISTING);
207+
}
208+
config = TikaLoader.load(tempConfig);
209+
} finally {
210+
Files.deleteIfExists(tempConfig);
204211
}
205212
}
206213
UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
207-
final TikaConfig finalConfig = config;
208-
SwingUtilities.invokeLater(() -> new TikaGUI(
209-
new DigestingParser(new AutoDetectParser(finalConfig),
210-
new CommonsDigester(MAX_MARK, CommonsDigester.DigestAlgorithm.MD5, CommonsDigester.DigestAlgorithm.SHA256),
211-
false), finalConfig).setVisible(true));
214+
final TikaLoader tikaLoader = config;
215+
SwingUtilities.invokeLater(() -> {
216+
try {
217+
new TikaGUI(tikaLoader.loadAutoDetectParser(), tikaLoader).setVisible(true);
218+
} catch (TikaConfigException e) {
219+
throw new RuntimeException(e);
220+
} catch (IOException e) {
221+
throw new RuntimeException(e);
222+
}
223+
});
212224
}
213225

214226
private void addMenuBar() {
@@ -384,7 +396,7 @@ private void handleStream(InputStream input, Metadata md) throws Exception {
384396
StringWriter jsonBuffer = new StringWriter();
385397
JsonMetadataList.setPrettyPrinting(true);
386398
List<Metadata> metadataList = recursiveParserWrapperHandler.getMetadataList();
387-
metadataList = tikaConfig.getMetadataFilter().filter(metadataList);
399+
metadataList = tikaConfig.loadMetadataFilters().filter(metadataList);
388400
JsonMetadataList.toJson(metadataList, jsonBuffer);
389401
setText(json, jsonBuffer.toString());
390402
}

‎tika-app/src/main/resources/tika-config-default-single-file.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@
77
"pdf-parser": {
88
"extractActions": true,
99
"extractInlineImages": true,
10-
"checkExtractAccessPermissions": true,
10+
"accessChecker": {
11+
"needToCheck": true,
12+
"allowExtractionForAccessibility": true
13+
},
1114
"extractIncrementalUpdateInfo": true,
1215
"parseIncrementalUpdates":true
1316

‎tika-app/src/main/resources/tika-config-default-single-file.xml‎

Lines changed: 0 additions & 49 deletions
This file was deleted.

‎tika-app/src/test/java/org/apache/tika/cli/TikaCLIAsyncTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ public void testAsync() throws Exception {
139139
json++;
140140
}
141141
}
142-
assertEquals(21, json);
142+
assertEquals(18, json);
143143
}
144144

145145
private void checkForPrettyPrint(File f) throws IOException {

‎tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.jetbrains.annotations.Nullable;
4343
import org.junit.jupiter.api.AfterEach;
4444
import org.junit.jupiter.api.BeforeEach;
45+
import org.junit.jupiter.api.Disabled;
4546
import org.junit.jupiter.api.Test;
4647
import org.junit.jupiter.api.io.TempDir;
4748

@@ -58,6 +59,7 @@
5859
public class TikaCLITest {
5960

6061
static final File TEST_DATA_FILE = new File("src/test/resources/test-data");
62+
static final File CONFIGS_DIR = new File("src/test/resources/configs");
6163
private final URI testDataURI = TEST_DATA_FILE.toURI();
6264
@TempDir
6365
private Path extractDir;
@@ -246,8 +248,7 @@ public void testJsonMetadataOutput() throws Exception {
246248
public void testJsonMetadataPrettyPrintOutput() throws Exception {
247249
String json = getParamOutContent("--json", "-r", resourcePrefix + "testJsonMultipleInts.html");
248250

249-
assertTrue(json.contains("\"X-TIKA:Parsed-By\" : [ \"org.apache.tika.parser.CompositeParser\", " +
250-
"\"org.apache.tika.parser.DefaultParser\", \"org.apache.tika.parser.html.JSoupParser\" ],"));
251+
assertTrue(json.contains("org.apache.tika.parser.CompositeParser\", \"org.apache.tika.parser.html.JSoupParser"));
251252
//test pretty-print alphabetic sort of keys
252253
int enc = json.indexOf("\"Content-Encoding\"");
253254
int fb = json.indexOf("fb:admins");
@@ -550,20 +551,11 @@ public void testDefaultConfigException() throws Exception {
550551

551552
@Test
552553
public void testConfig() throws Exception {
553-
String content = getParamOutContent("--config=" + TEST_DATA_FILE.toString() + "/tika-config1.xml", resourcePrefix + "bad_xml.xml");
554+
String content = getParamOutContent("--config=" + CONFIGS_DIR.toString() + "/tika-config1.json", resourcePrefix + "bad_xml.xml");
554555
assertTrue(content.contains("apple"));
555556
assertTrue(content.contains("org.apache.tika.parser.html.JSoupParser"));
556557
}
557558

558-
@Test
559-
public void testConfigIgnoreInit() throws Exception {
560-
String content = getParamOutContent("--config=" + TEST_DATA_FILE.toString() + "/TIKA-2389-ignore-init-problems.xml", resourcePrefix + "test_recursive_embedded.docx");
561-
assertTrue(content.contains("embed_1a"));
562-
//TODO: add a real unit test that configures logging to a file to test that nothing is
563-
//written at the various logging levels
564-
}
565-
566-
567559
@Test
568560
public void testJsonRecursiveMetadataParserMetadataOnly() throws Exception {
569561
String content = getParamOutContent("-m", "-J", "-r", resourcePrefix + "test_recursive_embedded.docx");
@@ -594,6 +586,7 @@ public void testDigestInJson() throws Exception {
594586
}
595587

596588
@Test
589+
@Disabled("until we re-implement serialization")
597590
public void testConfigSerializationStaticAndCurrent() throws Exception {
598591
String content = getParamOutContent("--dump-static-config");
599592
//make sure at least one detector is there
@@ -610,8 +603,9 @@ public void testConfigSerializationStaticAndCurrent() throws Exception {
610603
}
611604

612605
@Test
606+
@Disabled("until we re-implement serialization")
613607
public void testConfigSerializationCustomMinimal() throws Exception {
614-
String content = getParamOutContent("--config=" + TEST_DATA_FILE.toString() + "/tika-config2.xml", "--dump-minimal-config").replaceAll("[\r\n\t ]+", " ");
608+
String content = getParamOutContent("--config=" + CONFIGS_DIR.toString() + "/tika-config2.json", "--dump-minimal-config").replaceAll("[\r\n\t ]+", " ");
615609

616610
String expected =
617611
"<parser class=\"org.apache.tika.parser.DefaultParser\">" + " <mime-exclude>application/pdf</mime-exclude>" + " <mime-exclude>image/jpeg</mime-exclude> " +
@@ -620,8 +614,9 @@ public void testConfigSerializationCustomMinimal() throws Exception {
620614
}
621615

622616
@Test
617+
@Disabled("until we re-implement serialization")
623618
public void testConfigSerializationCustomStatic() throws Exception {
624-
String content = getParamOutContent("--config=" + TEST_DATA_FILE.toString() + "/tika-config2.xml", "--dump-static-config");
619+
String content = getParamOutContent("--config=" + TEST_DATA_FILE.toString() + "/tika-config2.json", "--dump-static-config");
625620
assertFalse(content.contains("org.apache.tika.parser.executable.Executable"));
626621
}
627622

‎tika-app/src/test/resources/configs/tika-config1.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"parsers": [
3+
{
4+
"jsoup-parser": {
5+
"_decorate": {
6+
"mimeInclude": [
7+
"application/vnd.wap.xhtml+xml",
8+
"application/x-asp",
9+
"application/xhtml+xml",
10+
"text/html",
11+
"application/xml",
12+
"text/xml"
13+
]
14+
}
15+
}
16+
}
17+
]
18+
}

‎tika-app/src/test/resources/configs/tika-config2.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"parsers": [
3+
{
4+
"default-parser": {
5+
"_decorate": {
6+
"mimeExclude": [
7+
"image/jpeg",
8+
"application/pdf"
9+
],
10+
"parserExclude": [
11+
"org.apache.tika.parser.executable.ExecutableParser"
12+
]
13+
}
14+
}
15+
},
16+
{
17+
"empty-parser": {
18+
"_decorate": {
19+
"mimeInclude": [
20+
"application/pdf"
21+
]
22+
}
23+
}
24+
}
25+
]
26+
}

0 commit comments

Comments
 (0)