Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 62 additions & 64 deletions tika-app/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -89,75 +89,16 @@
</resource>
</resources>
<plugins>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>${maven.shade.version}</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<createDependencyReducedPom>
false
</createDependencyReducedPom>
<artifactSet>
<excludes>
<exclude>org.apache.tika:tika-parsers-standard-package:jar:</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/maven/plugin.xml</exclude>
<exclude>module-info.class</exclude>
<exclude>META-INF/*</exclude>
<exclude>LICENSE.txt</exclude>
<exclude>NOTICE.txt</exclude>
<exclude>CHANGES</exclude>
<exclude>README</exclude>
<exclude>builddef.lst</exclude>
<!-- https://issues.apache.org/jira/browse/TIKA-3650 -->
<exclude>javax/**/*</exclude>

</excludes>
</filter>
</filters>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.apache.tika.cli.TikaCLI</mainClass>
<manifestEntries>
<Multi-Release>true</Multi-Release>
</manifestEntries>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/LICENSE</resource>
<file>target/classes/META-INF/LICENSE</file>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/NOTICE</resource>
<file>target/classes/META-INF/NOTICE</file>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/DEPENDENCIES</resource>
<file>target/classes/META-INF/DEPENDENCIES</file>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/cxf/bus-extensions.txt</resource>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<mainClass>org.apache.tika.cli.TikaCLI</mainClass>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
</manifest>
<manifestEntries>
<Automatic-Module-Name>org.apache.tika.app</Automatic-Module-Name>
</manifestEntries>
Expand All @@ -174,6 +115,63 @@
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>src/main/assembly/assembly.xml</descriptor>
</descriptors>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-plugins</id>
<phase>generate-test-resources</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/plugins</outputDirectory>
<artifactItems>
<artifactItem>
<groupId>org.apache.tika</groupId>
<artifactId>tika-fetcher-file-system</artifactId>
<version>${project.version}</version>
<type>jar</type>
<overWrite>true</overWrite>
</artifactItem>
</artifactItems>
</configuration>
</execution>
<!-- <execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<includeScope>runtime</includeScope>
<stripVersion>false</stripVersion>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
</configuration>
</execution> -->
</executions>
</plugin>
</plugins>
</build>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,27 +14,38 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
<assembly>
<id>standalone</id>
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.1 http://maven.apache.org/xsd/assembly-2.1.1.xsd">
<id>bin</id>
<formats>
<format>jar</format>
<format>zip</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>

<dependencySets>
<dependencySet>
<outputDirectory></outputDirectory>
<unpack>true</unpack>
<unpackOptions>
<excludes>
<exclude>META-INF/MANIFEST.MF</exclude>
<exclude>META-INF/README*</exclude>
<exclude>META-INF/NOTICE*</exclude>
<exclude>META-INF/LICENSE*</exclude>
<exclude>README*</exclude>
<exclude>NOTICE*</exclude>
<exclude>LICENSE*</exclude>
</excludes>
</unpackOptions>
<outputDirectory>lib</outputDirectory>
<useProjectArtifact>false</useProjectArtifact>
<unpack>false</unpack>
<scope>runtime</scope>
</dependencySet>
</dependencySets>
</assembly>
<fileSets>
<fileSet>
<directory>${project.build.directory}</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
<excludes>
<exclude>*-sources.jar</exclude>
<exclude>*-javadoc.jar</exclude>
</excludes>
</fileSet>
<fileSet>
<directory>${project.build.directory}/plugins</directory>
<outputDirectory>plugins</outputDirectory>
</fileSet>
</fileSets>
</assembly>
2 changes: 0 additions & 2 deletions tika-app/src/main/java/org/apache/tika/cli/AsyncHelper.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ public static String[] translateArgs(String[] args) {
String c = arg.substring(TIKA_CONFIG_KEY.length());
argList.add("-c");
argList.add(c);
} else if (arg.equals("-a")) {
//do nothing
} else {
argList.add(args[i]);
}
Expand Down
5 changes: 5 additions & 0 deletions tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,11 @@ private static void async(String[] args) throws Exception {
TikaAsyncCLI.main(args);
return;
}
if (args.length == 2 && args[0].endsWith(".xml") && args[1].endsWith(".json")) {
TikaAsyncCLI.main(args);
return;
};
//TODO -- are there other shortcuts?
Path tmpConfig = null;
try {
tmpConfig = Files.createTempFile("tika-config-", ".xml");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ public class AsyncHelperTest {

@Test
public void testBasic() throws Exception {
String[] args = new String[]{"-a", "--config=blah.xml", "-i", "input.docx", "-o", "output/dir"};
String[] expected = new String[]{"-c", "blah.xml", "-i", "input.docx", "-o", "output/dir"};
String[] args = new String[]{"-a", "blah.json", "--config=blah.xml", "-i", "input.docx", "-o", "output/dir"};
String[] expected = new String[]{"-a", "blah.json", "-c", "blah.xml", "-i", "input.docx", "-o", "output/dir"};
assertArrayEquals(expected, AsyncHelper.translateArgs(args));
}
}
38 changes: 31 additions & 7 deletions tika-app/src/test/java/org/apache/tika/cli/TikaCLIAsyncTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,34 @@
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TikaCLIAsyncTest {

private static final Logger LOG = LoggerFactory.getLogger(TikaCLI.class);

final static String JSON_TEMPLATE = """
{
"pipesPluginsConfig" : {
"fetchers": {
"file-system-fetcher": {
"basePath": "BASE_PATH",
"extractFileSystemMetadata": false
}
},
"pf4j.pluginsDir": "PLUGINS_DIR"
}
}
""";

static final File TEST_DATA_FILE = new File("src/test/resources/test-data");

Expand All @@ -46,20 +64,25 @@ public class TikaCLIAsyncTest {
private PrintStream stderr = null;

private static Path ASYNC_CONFIG;
private static Path ASYNC_PLUGINS_CONFIG;

@TempDir
private static Path ASYNC_OUTPUT_DIR;

@BeforeAll
public static void setUpClass() throws Exception {
ASYNC_CONFIG = Files.createTempFile(ASYNC_OUTPUT_DIR, "async-config-", ".xml");
String xml = "<properties>" + "<async>" + "<numClients>3</numClients>" + "<tikaConfig>" + ASYNC_CONFIG.toAbsolutePath() + "</tikaConfig>" + "</async>" + "<fetchers>" +
"<fetcher class=\"org.apache.tika.pipes.fetcher.fs.FileSystemFetcher\">" + "<name>fsf</name>" + "<basePath>" + TEST_DATA_FILE.getAbsolutePath() +
"</basePath>" +
"</fetcher>" + "</fetchers>" + "<emitters>" + "<emitter class=\"org.apache.tika.pipes.emitter.fs.FileSystemEmitter\">" + "<name>fse</name>" + "<basePath>" +
String xml = "<properties>" + "<async>" + "<numClients>3</numClients>" + "<tikaConfig>" + ASYNC_CONFIG.toAbsolutePath() + "</tikaConfig>" + "</async>" +
"<emitters>" + "<emitter class=\"org.apache.tika.pipes.emitter.fs.FileSystemEmitter\">" + "<name>fse</name>" + "<basePath>" +
ASYNC_OUTPUT_DIR.toAbsolutePath() + "</basePath>" + "<prettyPrint>true</prettyPrint>" + "</emitter>" + "</emitters>" +
"<pipesIterator class=\"org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator\">" + "<basePath>" + TEST_DATA_FILE.getAbsolutePath() + "</basePath>" +
"<fetcherName>fsf</fetcherName>" + "<emitterName>fse</emitterName>" + "</pipesIterator>" + "</properties>";
"<fetcherName>file-system-fetcher</fetcherName>" + "<emitterName>fse</emitterName>" + "</pipesIterator>" + "</properties>";
Files.write(ASYNC_CONFIG, xml.getBytes(UTF_8));
ASYNC_PLUGINS_CONFIG = Files.createTempFile(ASYNC_OUTPUT_DIR, "plugins-", ".json");

Path pluginsDir = Paths.get("target/plugins");
String json = JSON_TEMPLATE.replace("BASE_PATH", TEST_DATA_FILE.getAbsolutePath().toString()).replace("PLUGINS_DIR", pluginsDir.toAbsolutePath().toString());
Files.writeString(ASYNC_PLUGINS_CONFIG, json, UTF_8);
}

/**
Expand Down Expand Up @@ -103,7 +126,8 @@ private void resetContent() throws Exception {

@Test
public void testAsync() throws Exception {
String content = getParamOutContent("-a", "-c", ASYNC_CONFIG.toAbsolutePath().toString());
String content = getParamOutContent("-c", ASYNC_CONFIG.toAbsolutePath().toString(),
"-a", ASYNC_PLUGINS_CONFIG.toAbsolutePath().toString());

int json = 0;
for (File f : ASYNC_OUTPUT_DIR
Expand All @@ -121,7 +145,7 @@ public void testAsync() throws Exception {
json++;
}
}
assertEquals(20, json);
assertEquals(21, json);
}

private void checkForPrettyPrint(File f) throws IOException {
Expand Down
17 changes: 15 additions & 2 deletions tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.apache.tika.cli;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.tika.cli.TikaCLIAsyncTest.JSON_TEMPLATE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
Expand Down Expand Up @@ -368,13 +369,25 @@ private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFil

private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFileNames, int expectedLength) throws Exception {
Path input = Paths.get(new URI(resourcePrefix + "/" + targetFile));
Path asyncConfig = Files.createTempFile("async-config-", ".json");
Path pluginsDir = Paths.get("target/plugins");

String json = JSON_TEMPLATE.replace("BASE_PATH", TEST_DATA_FILE.getAbsolutePath().toString())
.replace("PLUGINS_DIR", pluginsDir.toAbsolutePath().toString());
Files.writeString(asyncConfig, json, UTF_8);

String[] params = {"-Z",
"-a", asyncConfig.toAbsolutePath().toString(),

ProcessUtils.escapeCommandLine(input.toAbsolutePath().toString()),
ProcessUtils.escapeCommandLine(extractDir
.toAbsolutePath()
.toString())};

TikaCLI.main(params);
try {
TikaCLI.main(params);
} finally {
Files.delete(asyncConfig);
}
Set<String> fileNames = getFileNames(extractDir);
String[] jsonFile = extractDir
.toFile()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
Expand All @@ -45,7 +47,7 @@ public abstract class ConfigBase {

private static Class[] SUPPORTED_PRIMITIVES =
new Class[]{String.class, boolean.class, long.class, int.class, double.class,
float.class};
float.class, Path.class};

/**
* Use this to build a single class, where the user specifies the instance class, e.g.
Expand Down Expand Up @@ -493,6 +495,8 @@ private static void tryToSetPrimitive(Object object, SetterClassPair setterClass
setterClassPair.setterMethod.invoke(object, Double.parseDouble(value));
} else if (setterClassPair.itemClass == boolean.class) {
setterClassPair.setterMethod.invoke(object, Boolean.parseBoolean(value));
} else if (setterClassPair.itemClass == Path.class) {
setterClassPair.setterMethod.invoke(object, Paths.get(value));
} else {
setterClassPair.setterMethod.invoke(object, value);
}
Expand Down
Loading