Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,10 @@ public class TikaComponentProcessor extends AbstractProcessor {
SERVICE_INTERFACES.put("org.apache.tika.parser.Parser", "parsers");
SERVICE_INTERFACES.put("org.apache.tika.detect.Detector", "detectors");
SERVICE_INTERFACES.put("org.apache.tika.detect.EncodingDetector", "encoding-detectors");
SERVICE_INTERFACES.put("org.apache.tika.language.detect.LanguageDetector", "language-detectors");
SERVICE_INTERFACES.put("org.apache.tika.language.translate.Translator", "translators");
SERVICE_INTERFACES.put("org.apache.tika.renderer.Renderer", "renderers");
SERVICE_INTERFACES.put("org.apache.tika.metadata.listfilter.MetadataFilter", "metadata-filters");
SERVICE_INTERFACES.put("org.apache.tika.metadata.filter.MetadataFilter", "metadata-filters");
}

private Messager messager;
Expand Down
2 changes: 2 additions & 0 deletions tika-app/src/main/java/org/apache/tika/cli/AsyncHelper.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ public static String[] translateArgs(String[] args) {
String c = arg.substring(TIKA_CONFIG_KEY.length());
argList.add("-c");
argList.add(c);
} else if ("-a".equals(arg)) {
//do nothing
} else {
argList.add(args[i]);
}
Expand Down
15 changes: 10 additions & 5 deletions tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
Original file line number Diff line number Diff line change
Expand Up @@ -260,25 +260,29 @@ public static void main(String[] args) throws Exception {
private static void async(String[] args) throws Exception {
args = AsyncHelper.translateArgs(args);
String tikaConfigPath = "";
//TODO - runpack is a smelly. fix this.
boolean runpack = false;
for (int i = 0; i < args.length - 1; i++) {
if (args[i].equals("-c")) {
tikaConfigPath = args[i + 1];
break;
} else if ("-Z".equals(args[i])) {
runpack = true;
}
}
if (! StringUtils.isBlank(tikaConfigPath)) {

if (runpack || ! StringUtils.isBlank(tikaConfigPath)) {
TikaAsyncCLI.main(args);
return;
}
if (args.length == 2 && args[0].endsWith(".xml") && args[1].endsWith(".json")) {
if (args.length == 1 && args[0].endsWith(".json")) {
TikaAsyncCLI.main(args);
return;
};
//TODO -- are there other shortcuts?
Path tmpConfig = null;
try {
tmpConfig = Files.createTempFile("tika-config-", ".xml");
Files.copy(TikaCLI.class.getResourceAsStream("/tika-config-default-single-file.xml"),
tmpConfig = Files.createTempFile("tika-config-", ".json");
Files.copy(TikaCLI.class.getResourceAsStream("/tika-config-default-single-file.json"),
tmpConfig, StandardCopyOption.REPLACE_EXISTING);
List<String> argList = new ArrayList<>();
argList.add("-c");
Expand Down Expand Up @@ -352,6 +356,7 @@ private boolean testForAsync(String[] args) {
return true;
}
}

for (String arg : args) {
if (arg.equals("-a") || arg.equals("--async")) {
return true;
Expand Down
29 changes: 29 additions & 0 deletions tika-app/src/main/resources/tika-config-default-single-file.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{
"parsers": [
{
"default-parser": {}
},
{
"pdf-parser": {
"extractActions": true,
"extractInlineImages": true,
"checkExtractAccessPermissions": true,
"extractIncrementalUpdateInfo": true,
"parseIncrementalUpdates":true

}
},
{
"ooxml-parser": {
"includeDeletedContent": true,
"includeMoveFromContent": true,
"extractMacros": true
}
},
{
"office-parser": {
"extractMacros": true
}
}
]
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ public class AsyncHelperTest {

@Test
public void testBasic() throws Exception {
String[] args = new String[]{"-a", "blah.json", "--config=blah.xml", "-i", "input.docx", "-o", "output/dir"};
String[] expected = new String[]{"-a", "blah.json", "-c", "blah.xml", "-i", "input.docx", "-o", "output/dir"};
String[] args = new String[]{"-a", "--config=blah.json", "-i", "input.docx", "-o", "output/dir"};
String[] expected = new String[]{"-c", "blah.json", "-i", "input.docx", "-o", "output/dir"};
assertArrayEquals(expected, AsyncHelper.translateArgs(args));
}
}
20 changes: 8 additions & 12 deletions tika-app/src/test/java/org/apache/tika/cli/TikaCLIAsyncTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -50,18 +50,14 @@ public class TikaCLIAsyncTest {
private PrintStream stdout = null;
private PrintStream stderr = null;

private static Path ASYNC_CONFIG;
private static Path ASYNC_PLUGINS_CONFIG;
private static Path TIKA_CONFIG;

@TempDir
private static Path ASYNC_OUTPUT_DIR;

@BeforeAll
public static void setUpClass() throws Exception {
ASYNC_CONFIG = Files.createTempFile(ASYNC_OUTPUT_DIR, "async-config-", ".xml");
String xml = "<properties/>";
Files.write(ASYNC_CONFIG, xml.getBytes(UTF_8));
ASYNC_PLUGINS_CONFIG = Files.createTempFile(ASYNC_OUTPUT_DIR, "plugins-", ".json");
TIKA_CONFIG = Files.createTempFile(ASYNC_OUTPUT_DIR, "plugins-", ".json");

Path pluginsDir = Paths.get("target/plugins");
if (! Files.isDirectory(pluginsDir)) {
Expand All @@ -73,12 +69,12 @@ public static void setUpClass() throws Exception {
String json = jsonTemplate.replace("FETCHER_BASE_PATH", TEST_DATA_FILE.getAbsolutePath().toString())
.replace("EMITTER_BASE_PATH", ASYNC_OUTPUT_DIR.toAbsolutePath().toString())
.replace("PLUGIN_ROOTS", pluginsDir.toAbsolutePath().toString())
.replace("PLUGINS_CONFIG", ASYNC_PLUGINS_CONFIG.toAbsolutePath().toString())
.replace("TIKA_CONFIG", ASYNC_CONFIG.toAbsolutePath().toString());
.replace("TIKA_CONFIG", TIKA_CONFIG
.toAbsolutePath().toString());

;
json = json.replace("\\", "/");
Files.writeString(ASYNC_PLUGINS_CONFIG, json, UTF_8);
Files.writeString(TIKA_CONFIG, json, UTF_8);
}

/**
Expand Down Expand Up @@ -124,8 +120,7 @@ private void resetContent() throws Exception {
public void testAsync() throws Exception {
//extension is "jsn" to avoid conflict with json config

String content = getParamOutContent("-c", ASYNC_CONFIG.toAbsolutePath().toString(),
"-a", ASYNC_PLUGINS_CONFIG.toAbsolutePath().toString());
String content = getParamOutContent("-a", "-c", TIKA_CONFIG.toAbsolutePath().toString());

int json = 0;
for (File f : ASYNC_OUTPUT_DIR
Expand All @@ -138,7 +133,8 @@ public void testAsync() throws Exception {
if (f
.getName()
.equals("coffee.xls.jsn")) {
checkForPrettyPrint(f);
//TODO -- turn this back on
// checkForPrettyPrint(f);
}
json++;
}
Expand Down
1 change: 0 additions & 1 deletion tika-app/src/test/resources/configs/config-template.json
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@
"staleFetcherDelaySeconds": 60,
"forkedJvmArgs": ["-Xmx1g", "-XX:+UseG1GC"],
"tikaConfig": "TIKA_CONFIG",
"pipesPluginsConfig": "PLUGINS_CONFIG",
"javaPath": "java"
},
"plugin-roots": "PLUGIN_ROOTS"
Expand Down
4 changes: 4 additions & 0 deletions tika-core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,10 @@
<Export-Package>
org.apache.tika.*
</Export-Package>
<Include-Resource>
{maven-resources},
META-INF/tika=${project.build.outputDirectory}/META-INF/tika
</Include-Resource>
</instructions>
</configuration>
</plugin>
Expand Down
41 changes: 40 additions & 1 deletion tika-core/src/main/java/org/apache/tika/config/ConfigBase.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;

import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
Expand All @@ -47,7 +48,7 @@ public abstract class ConfigBase {

private static Class[] SUPPORTED_PRIMITIVES =
new Class[]{String.class, boolean.class, long.class, int.class, double.class,
float.class, Path.class};
float.class, Path.class, Integer.class, Long.class, Double.class, Float.class, Boolean.class};

/**
* Use this to build a single class, where the user specifies the instance class, e.g.
Expand Down Expand Up @@ -253,6 +254,9 @@ private static void setParams(Object object, Node targetNode, Set<String> settin
} else if (setterClassPair.itemClass.isAssignableFrom(List.class)) {
tryToSetList(object, param);
processed = true;
} else if (setterClassPair.itemClass.isAssignableFrom(Set.class)) {
tryToSetSet(object, param);
processed = true;
}
}
if (!processed) {
Expand Down Expand Up @@ -359,6 +363,33 @@ private static boolean hasChildNodes(Node param) {
return false;
}

private static void tryToSetSet(Object object, Node param) throws TikaConfigException {
//simple hack for now -- only handle Set<String>
tryToSetStringSet(object, param);
}

private static void tryToSetStringSet(Object object, Node param) throws TikaConfigException {
String name = param.getLocalName();
Set<String> strings = new TreeSet<>();
NodeList nodeList = param.getChildNodes();
for (int i = 0; i < nodeList.getLength(); i++) {
Node n = nodeList.item(i);
if (n.getNodeType() == 1) {
String txt = n.getTextContent();
if (txt != null) {
strings.add(txt);
}
}
}
String setter = "set" + name.substring(0, 1).toUpperCase(Locale.US) + name.substring(1);
try {
Method m = object.getClass().getMethod(setter, Set.class);
m.invoke(object, strings);
} catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
throw new TikaConfigException("can't set " + name, e);
}
}

private static void tryToSetList(Object object, Node param) throws TikaConfigException {
if (hasClass(param)) {
tryToSetClassList(object, param);
Expand Down Expand Up @@ -495,6 +526,14 @@ private static void tryToSetPrimitive(Object object, SetterClassPair setterClass
setterClassPair.setterMethod.invoke(object, Double.parseDouble(value));
} else if (setterClassPair.itemClass == boolean.class) {
setterClassPair.setterMethod.invoke(object, Boolean.parseBoolean(value));
} else if (setterClassPair.itemClass == Long.class) {
setterClassPair.setterMethod.invoke(object, Long.parseLong(value));
} else if (setterClassPair.itemClass == Float.class) {
setterClassPair.setterMethod.invoke(object, Float.parseFloat(value));
} else if (setterClassPair.itemClass == Double.class) {
setterClassPair.setterMethod.invoke(object, Double.parseDouble(value));
} else if (setterClassPair.itemClass == Boolean.class) {
setterClassPair.setterMethod.invoke(object, Boolean.parseBoolean(value));
} else if (setterClassPair.itemClass == Path.class) {
setterClassPair.setterMethod.invoke(object, Paths.get(value));
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,20 +57,23 @@ public void set(String name, String value) {
}
}

public <T> Optional<String> get(Class<T> key) {
return Optional.ofNullable(configs.get(key.getName()));
public <T> Optional<JsonConfig> get(Class<T> key) {
String json = configs.get(key.getName());
return json == null ? Optional.empty() : Optional.of(() -> json);
}

public Optional<String> get(String key) {
return Optional.ofNullable(configs.get(key));
public Optional<JsonConfig> get(String key) {
String json = configs.get(key);
return json == null ? Optional.empty() : Optional.of(() -> json);
}

public String get(String key, String defaultMissing) {
public JsonConfig get(String key, String defaultMissing) {
String val = configs.get(key);
if (val == null) {
return defaultMissing;
val = defaultMissing;
}
return val;
final String jsonValue = val;
return () -> jsonValue;
}

public Set<String> getKeys() {
Expand Down
Loading