TIKA-4519 -- fully integrate pf4j in pipes components

pom.xml

@@ -39,12 +39,14 @@
     <module>tika-bom</module>
     <module>tika-core</module>
     <module>tika-serialization</module>
+    <module>tika-plugins-core</module>
     <module>tika-detectors</module>
     <module>tika-parsers</module>
     <module>tika-bundles</module>
     <module>tika-xmp</module>
     <module>tika-langdetect</module>
     <module>tika-pipes</module>
+
     <module>tika-grpc</module>
     <module>tika-app</module>
     <module>tika-server</module>

tika-app/pom.xml

@@ -89,75 +89,16 @@
       </resource>
     </resources>
     <plugins>
-      <plugin>
-        <artifactId>maven-shade-plugin</artifactId>
-        <version>${maven.shade.version}</version>
-        <executions>
-          <execution>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <configuration>
-              <createDependencyReducedPom>
-                false
-              </createDependencyReducedPom>
-              <artifactSet>
-                <excludes>
-                  <exclude>org.apache.tika:tika-parsers-standard-package:jar:</exclude>
-                </excludes>
-              </artifactSet>
-              <filters>
-                <filter>
-                  <artifact>*:*</artifact>
-                  <excludes>
-                    <exclude>META-INF/maven/plugin.xml</exclude>
-                    <exclude>module-info.class</exclude>
-                    <exclude>META-INF/*</exclude>
-                    <exclude>LICENSE.txt</exclude>
-                    <exclude>NOTICE.txt</exclude>
-                    <exclude>CHANGES</exclude>
-                    <exclude>README</exclude>
-                    <exclude>builddef.lst</exclude>
-                    <!-- https://issues.apache.org/jira/browse/TIKA-3650 -->
-                    <exclude>javax/**/*</exclude>
-
-                  </excludes>
-                </filter>
-              </filters>
-              <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                  <mainClass>org.apache.tika.cli.TikaCLI</mainClass>
-                  <manifestEntries>
-                    <Multi-Release>true</Multi-Release>
-                  </manifestEntries>
-                </transformer>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
-                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
-                  <resource>META-INF/LICENSE</resource>
-                  <file>target/classes/META-INF/LICENSE</file>
-                </transformer>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
-                  <resource>META-INF/NOTICE</resource>
-                  <file>target/classes/META-INF/NOTICE</file>
-                </transformer>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
-                  <resource>META-INF/DEPENDENCIES</resource>
-                  <file>target/classes/META-INF/DEPENDENCIES</file>
-                </transformer>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
-                  <resource>META-INF/cxf/bus-extensions.txt</resource>
-                </transformer>
-              </transformers>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
         <configuration>
           <archive>
+            <manifest>
+              <mainClass>org.apache.tika.cli.TikaCLI</mainClass>
+              <addClasspath>true</addClasspath>
+              <classpathPrefix>lib/</classpathPrefix>
+            </manifest>
             <manifestEntries>
               <Automatic-Module-Name>org.apache.tika.app</Automatic-Module-Name>
             </manifestEntries>
@@ -174,6 +115,77 @@
           </excludes>
         </configuration>
       </plugin>
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <configuration>
+          <descriptors>
+            <descriptor>src/main/assembly/assembly.xml</descriptor>
+          </descriptors>
+          <appendAssemblyId>false</appendAssemblyId>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>copy-plugins</id>
+            <phase>process-test-resources</phase>
+            <goals>
+              <goal>copy</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/plugins</outputDirectory>
+              <artifactItems>
+                <artifactItem>
+                  <groupId>org.apache.tika</groupId>
+                  <artifactId>tika-fetcher-file-system</artifactId>
+                  <version>${project.version}</version>
+                  <type>zip</type>
+                  <overWrite>true</overWrite>
+                </artifactItem>
+                <artifactItem>
+                  <groupId>org.apache.tika</groupId>
+                  <artifactId>tika-emitter-file-system</artifactId>
+                  <version>${project.version}</version>
+                  <type>zip</type>
+                  <overWrite>true</overWrite>
+                </artifactItem>
+                <artifactItem>
+                  <groupId>org.apache.tika</groupId>
+                  <artifactId>tika-pipes-iterator-file-system</artifactId>
+                  <version>${project.version}</version>
+                  <type>zip</type>
+                  <overWrite>true</overWrite>
+                </artifactItem>
+              </artifactItems>
+            </configuration>
+          </execution>
+          <execution>
+            <id>copy-dependencies</id>
+            <phase>package</phase>
+            <goals>
+              <goal>copy-dependencies</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/lib</outputDirectory>
+              <includeScope>runtime</includeScope>
+              <stripVersion>false</stripVersion>
+              <overWriteReleases>false</overWriteReleases>
+              <overWriteSnapshots>false</overWriteSnapshots>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

tika-app/src/main/assembly/standalone.xml → tika-app/src/main/assembly/assembly.xml

@@ -14,27 +14,38 @@
  See the License for the specific language governing permissions and
  limitations under the License.
 -->
-<assembly>
-  <id>standalone</id>
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.1"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.1 http://maven.apache.org/xsd/assembly-2.1.1.xsd">
+  <id>bin</id>
   <formats>
-    <format>jar</format>
+    <format>zip</format>
   </formats>
   <includeBaseDirectory>false</includeBaseDirectory>
+
   <dependencySets>
     <dependencySet>
-      <outputDirectory></outputDirectory>
-      <unpack>true</unpack>
-      <unpackOptions>
-        <excludes>
-          <exclude>META-INF/MANIFEST.MF</exclude>
-          <exclude>META-INF/README*</exclude>
-          <exclude>META-INF/NOTICE*</exclude>
-          <exclude>META-INF/LICENSE*</exclude>
-          <exclude>README*</exclude>
-          <exclude>NOTICE*</exclude>
-          <exclude>LICENSE*</exclude>
-        </excludes>
-      </unpackOptions>
+      <outputDirectory>lib</outputDirectory>
+      <useProjectArtifact>false</useProjectArtifact>
+      <unpack>false</unpack>
+      <scope>runtime</scope>
     </dependencySet>
   </dependencySets>
-</assembly>
+  <fileSets>
+    <fileSet>
+      <directory>${project.build.directory}</directory>
+      <outputDirectory>/</outputDirectory>
+      <includes>
+        <include>*.jar</include>
+      </includes>
+      <excludes>
+        <exclude>*-sources.jar</exclude>
+        <exclude>*-javadoc.jar</exclude>
+      </excludes>
+    </fileSet>
+    <fileSet>
+      <directory>${project.build.directory}/plugins</directory>
+      <outputDirectory>plugins</outputDirectory>
+    </fileSet>
+  </fileSets>
+</assembly>

tika-app/src/main/java/org/apache/tika/cli/AsyncHelper.java

@@ -32,8 +32,6 @@ public static String[] translateArgs(String[] args) {
                 String c = arg.substring(TIKA_CONFIG_KEY.length());
                 argList.add("-c");
                 argList.add(c);
-            } else if (arg.equals("-a")) {
-                //do nothing
             } else {
                 argList.add(args[i]);
             }

tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java

@@ -270,6 +270,11 @@ private static void async(String[] args) throws Exception {
             TikaAsyncCLI.main(args);
             return;
         }
+        if (args.length == 2 && args[0].endsWith(".xml") &&  args[1].endsWith(".json")) {
+            TikaAsyncCLI.main(args);
+            return;
+        };
+        //TODO -- are there other shortcuts?
         Path tmpConfig = null;
         try {
             tmpConfig = Files.createTempFile("tika-config-", ".xml");

tika-app/src/test/java/org/apache/tika/cli/AsyncHelperTest.java

@@ -24,8 +24,8 @@ public class AsyncHelperTest {
 
     @Test
     public void testBasic() throws Exception {
-        String[] args = new String[]{"-a", "--config=blah.xml", "-i", "input.docx", "-o", "output/dir"};
-        String[] expected = new String[]{"-c", "blah.xml", "-i", "input.docx", "-o", "output/dir"};
+        String[] args = new String[]{"-a", "blah.json", "--config=blah.xml", "-i", "input.docx", "-o", "output/dir"};
+        String[] expected = new String[]{"-a", "blah.json", "-c", "blah.xml", "-i", "input.docx", "-o", "output/dir"};
         assertArrayEquals(expected, AsyncHelper.translateArgs(args));
     }
 }

tika-app/src/test/java/org/apache/tika/cli/TikaCLIAsyncTest.java

@@ -24,18 +24,23 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 
 import org.apache.commons.io.FileUtils;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TikaCLIAsyncTest {
 
+    private static final Logger LOG = LoggerFactory.getLogger(TikaCLI.class);
 
     static final File TEST_DATA_FILE = new File("src/test/resources/test-data");
 
@@ -46,20 +51,33 @@ public class TikaCLIAsyncTest {
     private PrintStream stderr = null;
 
     private static Path ASYNC_CONFIG;
+    private static Path ASYNC_PLUGINS_CONFIG;
+
     @TempDir
     private static Path ASYNC_OUTPUT_DIR;
 
     @BeforeAll
     public static void setUpClass() throws Exception {
         ASYNC_CONFIG = Files.createTempFile(ASYNC_OUTPUT_DIR, "async-config-", ".xml");
-        String xml = "<properties>" + "<async>" + "<numClients>3</numClients>" + "<tikaConfig>" + ASYNC_CONFIG.toAbsolutePath() + "</tikaConfig>" + "</async>" + "<fetchers>" +
-                "<fetcher class=\"org.apache.tika.pipes.fetcher.fs.FileSystemFetcher\">" + "<name>fsf</name>" + "<basePath>" + TEST_DATA_FILE.getAbsolutePath() +
-                "</basePath>" +
-                "</fetcher>" + "</fetchers>" + "<emitters>" + "<emitter class=\"org.apache.tika.pipes.emitter.fs.FileSystemEmitter\">" + "<name>fse</name>" + "<basePath>" +
-                ASYNC_OUTPUT_DIR.toAbsolutePath() + "</basePath>" + "<prettyPrint>true</prettyPrint>" + "</emitter>" + "</emitters>" +
-                "<pipesIterator class=\"org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator\">" + "<basePath>" + TEST_DATA_FILE.getAbsolutePath() + "</basePath>" +
-                "<fetcherName>fsf</fetcherName>" + "<emitterName>fse</emitterName>" + "</pipesIterator>" + "</properties>";
+        String xml = "<properties/>";
         Files.write(ASYNC_CONFIG, xml.getBytes(UTF_8));
+        ASYNC_PLUGINS_CONFIG = Files.createTempFile(ASYNC_OUTPUT_DIR, "plugins-", ".json");
+
+        Path pluginsDir = Paths.get("target/plugins");
+        if (! Files.isDirectory(pluginsDir)) {
+            LOG.warn("CAN'T FIND PLUGINS DIR. pwd={}", Paths.get("").toAbsolutePath().toString());
+        }
+        String jsonTemplate = Files.readString(Paths.get(TikaCLIAsyncTest.class.getResource("/configs/config-template.json").toURI()),
+                StandardCharsets.UTF_8);
+
+        String json = jsonTemplate.replace("FETCHER_BASE_PATH", TEST_DATA_FILE.getAbsolutePath().toString())
+                                   .replace("EMITTER_BASE_PATH", ASYNC_OUTPUT_DIR.toAbsolutePath().toString())
+                                   .replace("PLUGIN_ROOTS", pluginsDir.toAbsolutePath().toString())
+                .replace("PLUGINS_CONFIG", ASYNC_PLUGINS_CONFIG.toAbsolutePath().toString())
+                        .replace("TIKA_CONFIG", ASYNC_CONFIG.toAbsolutePath().toString());
+
+                ;
+        Files.writeString(ASYNC_PLUGINS_CONFIG, json, UTF_8);
     }
 
     /**
@@ -103,19 +121,22 @@ private void resetContent() throws Exception {
 
     @Test
     public void testAsync() throws Exception {
-        String content = getParamOutContent("-a", "-c", ASYNC_CONFIG.toAbsolutePath().toString());
+        //extension is "jsn" to avoid conflict with json config
+
+        String content = getParamOutContent("-c", ASYNC_CONFIG.toAbsolutePath().toString(),
+                "-a", ASYNC_PLUGINS_CONFIG.toAbsolutePath().toString());
 
         int json = 0;
         for (File f : ASYNC_OUTPUT_DIR
                 .toFile()
                 .listFiles()) {
             if (f
                     .getName()
-                    .endsWith(".json")) {
+                    .endsWith(".jsn")) {
                 //check one file for pretty print
                 if (f
                         .getName()
-                        .equals("coffee.xls.json")) {
+                        .equals("coffee.xls.jsn")) {
                     checkForPrettyPrint(f);
                 }
                 json++;

tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java

@@ -281,6 +281,8 @@ public void testMacros() throws Exception {
 
     @Test
     public void testRUnpack() throws Exception {
+        //TODO -- rework this to use two separate emitters
+        //one for bytes and one for json
         String[] expectedChildren = new String[]{
                 "testPDFPackage.pdf.json",
                 //the first two test that the default single file config is working
@@ -396,10 +398,17 @@ private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFil
 
     private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFileNames, int expectedLength) throws Exception {
         Path input = Paths.get(new URI(resourcePrefix + "/" + targetFile));
-        String[] params = {"-Z", input.toAbsolutePath().toString(),
-                extractDir.toAbsolutePath().toString()};
+        Path pluginsDir = Paths.get("target/plugins");
+
+        String[] params = {"-Z",
+                "-p", ProcessUtils.escapeCommandLine(pluginsDir.toAbsolutePath().toString()),
+                ProcessUtils.escapeCommandLine(input.toAbsolutePath().toString()),
+                ProcessUtils.escapeCommandLine(extractDir
+                .toAbsolutePath()
+                .toString())};
 
         TikaCLI.main(params);
+
         Set<String> fileNames = getFileNames(extractDir);
         String[] jsonFile = extractDir
                 .toFile()
@@ -408,7 +417,7 @@ private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFil
         assertEquals(expectedLength, jsonFile.length);
 
         for (String expectedChildName : expectedChildrenFileNames) {
-            assertTrue(fileNames.contains(expectedChildName));
+            assertTrue(fileNames.contains(expectedChildName), expectedChildName);
         }
     }