Skip to content

Commit 256e2a1

Browse files
tballisontballison
authored
TIKA-4519 -- fully integrate pf4j in pipes components (#2385)
Generated-by: Claude Sonnet 4.5 (claude-sonnet-4-5-20250929) Starting with c43e0a4, the code was heavily developed/modified by Claude with substantial manual review and tweaks.
1 parent 10e93d2 commit 256e2a1

454 files changed

Lines changed: 16872 additions & 8795 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎pom.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,14 @@
3939
<module>tika-bom</module>
4040
<module>tika-core</module>
4141
<module>tika-serialization</module>
42+
<module>tika-plugins-core</module>
4243
<module>tika-detectors</module>
4344
<module>tika-parsers</module>
4445
<module>tika-bundles</module>
4546
<module>tika-xmp</module>
4647
<module>tika-langdetect</module>
4748
<module>tika-pipes</module>
49+
4850
<module>tika-grpc</module>
4951
<module>tika-app</module>
5052
<module>tika-server</module>

‎tika-app/pom.xml‎

Lines changed: 76 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -89,75 +89,16 @@
8989
</resource>
9090
</resources>
9191
<plugins>
92-
<plugin>
93-
<artifactId>maven-shade-plugin</artifactId>
94-
<version>${maven.shade.version}</version>
95-
<executions>
96-
<execution>
97-
<phase>package</phase>
98-
<goals>
99-
<goal>shade</goal>
100-
</goals>
101-
<configuration>
102-
<createDependencyReducedPom>
103-
false
104-
</createDependencyReducedPom>
105-
<artifactSet>
106-
<excludes>
107-
<exclude>org.apache.tika:tika-parsers-standard-package:jar:</exclude>
108-
</excludes>
109-
</artifactSet>
110-
<filters>
111-
<filter>
112-
<artifact>*:*</artifact>
113-
<excludes>
114-
<exclude>META-INF/maven/plugin.xml</exclude>
115-
<exclude>module-info.class</exclude>
116-
<exclude>META-INF/*</exclude>
117-
<exclude>LICENSE.txt</exclude>
118-
<exclude>NOTICE.txt</exclude>
119-
<exclude>CHANGES</exclude>
120-
<exclude>README</exclude>
121-
<exclude>builddef.lst</exclude>
122-
<!-- https://issues.apache.org/jira/browse/TIKA-3650 -->
123-
<exclude>javax/**/*</exclude>
124-
125-
</excludes>
126-
</filter>
127-
</filters>
128-
<transformers>
129-
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
130-
<mainClass>org.apache.tika.cli.TikaCLI</mainClass>
131-
<manifestEntries>
132-
<Multi-Release>true</Multi-Release>
133-
</manifestEntries>
134-
</transformer>
135-
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
136-
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
137-
<resource>META-INF/LICENSE</resource>
138-
<file>target/classes/META-INF/LICENSE</file>
139-
</transformer>
140-
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
141-
<resource>META-INF/NOTICE</resource>
142-
<file>target/classes/META-INF/NOTICE</file>
143-
</transformer>
144-
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
145-
<resource>META-INF/DEPENDENCIES</resource>
146-
<file>target/classes/META-INF/DEPENDENCIES</file>
147-
</transformer>
148-
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
149-
<resource>META-INF/cxf/bus-extensions.txt</resource>
150-
</transformer>
151-
</transformers>
152-
</configuration>
153-
</execution>
154-
</executions>
155-
</plugin>
15692
<plugin>
15793
<groupId>org.apache.maven.plugins</groupId>
15894
<artifactId>maven-jar-plugin</artifactId>
15995
<configuration>
16096
<archive>
97+
<manifest>
98+
<mainClass>org.apache.tika.cli.TikaCLI</mainClass>
99+
<addClasspath>true</addClasspath>
100+
<classpathPrefix>lib/</classpathPrefix>
101+
</manifest>
161102
<manifestEntries>
162103
<Automatic-Module-Name>org.apache.tika.app</Automatic-Module-Name>
163104
</manifestEntries>
@@ -174,6 +115,77 @@
174115
</excludes>
175116
</configuration>
176117
</plugin>
118+
<plugin>
119+
<artifactId>maven-assembly-plugin</artifactId>
120+
<configuration>
121+
<descriptors>
122+
<descriptor>src/main/assembly/assembly.xml</descriptor>
123+
</descriptors>
124+
<appendAssemblyId>false</appendAssemblyId>
125+
</configuration>
126+
<executions>
127+
<execution>
128+
<id>make-assembly</id>
129+
<phase>package</phase>
130+
<goals>
131+
<goal>single</goal>
132+
</goals>
133+
</execution>
134+
</executions>
135+
</plugin>
136+
<plugin>
137+
<groupId>org.apache.maven.plugins</groupId>
138+
<artifactId>maven-dependency-plugin</artifactId>
139+
<executions>
140+
<execution>
141+
<id>copy-plugins</id>
142+
<phase>process-test-resources</phase>
143+
<goals>
144+
<goal>copy</goal>
145+
</goals>
146+
<configuration>
147+
<outputDirectory>${project.build.directory}/plugins</outputDirectory>
148+
<artifactItems>
149+
<artifactItem>
150+
<groupId>org.apache.tika</groupId>
151+
<artifactId>tika-fetcher-file-system</artifactId>
152+
<version>${project.version}</version>
153+
<type>zip</type>
154+
<overWrite>true</overWrite>
155+
</artifactItem>
156+
<artifactItem>
157+
<groupId>org.apache.tika</groupId>
158+
<artifactId>tika-emitter-file-system</artifactId>
159+
<version>${project.version}</version>
160+
<type>zip</type>
161+
<overWrite>true</overWrite>
162+
</artifactItem>
163+
<artifactItem>
164+
<groupId>org.apache.tika</groupId>
165+
<artifactId>tika-pipes-iterator-file-system</artifactId>
166+
<version>${project.version}</version>
167+
<type>zip</type>
168+
<overWrite>true</overWrite>
169+
</artifactItem>
170+
</artifactItems>
171+
</configuration>
172+
</execution>
173+
<execution>
174+
<id>copy-dependencies</id>
175+
<phase>package</phase>
176+
<goals>
177+
<goal>copy-dependencies</goal>
178+
</goals>
179+
<configuration>
180+
<outputDirectory>${project.build.directory}/lib</outputDirectory>
181+
<includeScope>runtime</includeScope>
182+
<stripVersion>false</stripVersion>
183+
<overWriteReleases>false</overWriteReleases>
184+
<overWriteSnapshots>false</overWriteSnapshots>
185+
</configuration>
186+
</execution>
187+
</executions>
188+
</plugin>
177189
</plugins>
178190
</build>
179191

tika-app/src/main/assembly/standalone.xml renamed to tika-app/src/main/assembly/assembly.xml

Lines changed: 28 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,27 +14,38 @@
1414
See the License for the specific language governing permissions and
1515
limitations under the License.
1616
-->
17-
<assembly>
18-
<id>standalone</id>
17+
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.1"
18+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.1 http://maven.apache.org/xsd/assembly-2.1.1.xsd">
20+
<id>bin</id>
1921
<formats>
20-
<format>jar</format>
22+
<format>zip</format>
2123
</formats>
2224
<includeBaseDirectory>false</includeBaseDirectory>
25+
2326
<dependencySets>
2427
<dependencySet>
25-
<outputDirectory></outputDirectory>
26-
<unpack>true</unpack>
27-
<unpackOptions>
28-
<excludes>
29-
<exclude>META-INF/MANIFEST.MF</exclude>
30-
<exclude>META-INF/README*</exclude>
31-
<exclude>META-INF/NOTICE*</exclude>
32-
<exclude>META-INF/LICENSE*</exclude>
33-
<exclude>README*</exclude>
34-
<exclude>NOTICE*</exclude>
35-
<exclude>LICENSE*</exclude>
36-
</excludes>
37-
</unpackOptions>
28+
<outputDirectory>lib</outputDirectory>
29+
<useProjectArtifact>false</useProjectArtifact>
30+
<unpack>false</unpack>
31+
<scope>runtime</scope>
3832
</dependencySet>
3933
</dependencySets>
40-
</assembly>
34+
<fileSets>
35+
<fileSet>
36+
<directory>${project.build.directory}</directory>
37+
<outputDirectory>/</outputDirectory>
38+
<includes>
39+
<include>*.jar</include>
40+
</includes>
41+
<excludes>
42+
<exclude>*-sources.jar</exclude>
43+
<exclude>*-javadoc.jar</exclude>
44+
</excludes>
45+
</fileSet>
46+
<fileSet>
47+
<directory>${project.build.directory}/plugins</directory>
48+
<outputDirectory>plugins</outputDirectory>
49+
</fileSet>
50+
</fileSets>
51+
</assembly>

‎tika-app/src/main/java/org/apache/tika/cli/AsyncHelper.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ public static String[] translateArgs(String[] args) {
3232
String c = arg.substring(TIKA_CONFIG_KEY.length());
3333
argList.add("-c");
3434
argList.add(c);
35-
} else if (arg.equals("-a")) {
36-
//do nothing
3735
} else {
3836
argList.add(args[i]);
3937
}

‎tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,11 @@ private static void async(String[] args) throws Exception {
270270
TikaAsyncCLI.main(args);
271271
return;
272272
}
273+
if (args.length == 2 && args[0].endsWith(".xml") && args[1].endsWith(".json")) {
274+
TikaAsyncCLI.main(args);
275+
return;
276+
};
277+
//TODO -- are there other shortcuts?
273278
Path tmpConfig = null;
274279
try {
275280
tmpConfig = Files.createTempFile("tika-config-", ".xml");

‎tika-app/src/test/java/org/apache/tika/cli/AsyncHelperTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ public class AsyncHelperTest {
2424

2525
@Test
2626
public void testBasic() throws Exception {
27-
String[] args = new String[]{"-a", "--config=blah.xml", "-i", "input.docx", "-o", "output/dir"};
28-
String[] expected = new String[]{"-c", "blah.xml", "-i", "input.docx", "-o", "output/dir"};
27+
String[] args = new String[]{"-a", "blah.json", "--config=blah.xml", "-i", "input.docx", "-o", "output/dir"};
28+
String[] expected = new String[]{"-a", "blah.json", "-c", "blah.xml", "-i", "input.docx", "-o", "output/dir"};
2929
assertArrayEquals(expected, AsyncHelper.translateArgs(args));
3030
}
3131
}

‎tika-app/src/test/java/org/apache/tika/cli/TikaCLIAsyncTest.java‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,23 @@
2424
import java.io.File;
2525
import java.io.IOException;
2626
import java.io.PrintStream;
27+
import java.nio.charset.StandardCharsets;
2728
import java.nio.file.Files;
2829
import java.nio.file.Path;
30+
import java.nio.file.Paths;
2931

3032
import org.apache.commons.io.FileUtils;
3133
import org.junit.jupiter.api.AfterEach;
3234
import org.junit.jupiter.api.BeforeAll;
3335
import org.junit.jupiter.api.BeforeEach;
3436
import org.junit.jupiter.api.Test;
3537
import org.junit.jupiter.api.io.TempDir;
38+
import org.slf4j.Logger;
39+
import org.slf4j.LoggerFactory;
3640

3741
public class TikaCLIAsyncTest {
3842

43+
private static final Logger LOG = LoggerFactory.getLogger(TikaCLI.class);
3944

4045
static final File TEST_DATA_FILE = new File("src/test/resources/test-data");
4146

@@ -46,20 +51,33 @@ public class TikaCLIAsyncTest {
4651
private PrintStream stderr = null;
4752

4853
private static Path ASYNC_CONFIG;
54+
private static Path ASYNC_PLUGINS_CONFIG;
55+
4956
@TempDir
5057
private static Path ASYNC_OUTPUT_DIR;
5158

5259
@BeforeAll
5360
public static void setUpClass() throws Exception {
5461
ASYNC_CONFIG = Files.createTempFile(ASYNC_OUTPUT_DIR, "async-config-", ".xml");
55-
String xml = "<properties>" + "<async>" + "<numClients>3</numClients>" + "<tikaConfig>" + ASYNC_CONFIG.toAbsolutePath() + "</tikaConfig>" + "</async>" + "<fetchers>" +
56-
"<fetcher class=\"org.apache.tika.pipes.fetcher.fs.FileSystemFetcher\">" + "<name>fsf</name>" + "<basePath>" + TEST_DATA_FILE.getAbsolutePath() +
57-
"</basePath>" +
58-
"</fetcher>" + "</fetchers>" + "<emitters>" + "<emitter class=\"org.apache.tika.pipes.emitter.fs.FileSystemEmitter\">" + "<name>fse</name>" + "<basePath>" +
59-
ASYNC_OUTPUT_DIR.toAbsolutePath() + "</basePath>" + "<prettyPrint>true</prettyPrint>" + "</emitter>" + "</emitters>" +
60-
"<pipesIterator class=\"org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator\">" + "<basePath>" + TEST_DATA_FILE.getAbsolutePath() + "</basePath>" +
61-
"<fetcherName>fsf</fetcherName>" + "<emitterName>fse</emitterName>" + "</pipesIterator>" + "</properties>";
62+
String xml = "<properties/>";
6263
Files.write(ASYNC_CONFIG, xml.getBytes(UTF_8));
64+
ASYNC_PLUGINS_CONFIG = Files.createTempFile(ASYNC_OUTPUT_DIR, "plugins-", ".json");
65+
66+
Path pluginsDir = Paths.get("target/plugins");
67+
if (! Files.isDirectory(pluginsDir)) {
68+
LOG.warn("CAN'T FIND PLUGINS DIR. pwd={}", Paths.get("").toAbsolutePath().toString());
69+
}
70+
String jsonTemplate = Files.readString(Paths.get(TikaCLIAsyncTest.class.getResource("/configs/config-template.json").toURI()),
71+
StandardCharsets.UTF_8);
72+
73+
String json = jsonTemplate.replace("FETCHER_BASE_PATH", TEST_DATA_FILE.getAbsolutePath().toString())
74+
.replace("EMITTER_BASE_PATH", ASYNC_OUTPUT_DIR.toAbsolutePath().toString())
75+
.replace("PLUGIN_ROOTS", pluginsDir.toAbsolutePath().toString())
76+
.replace("PLUGINS_CONFIG", ASYNC_PLUGINS_CONFIG.toAbsolutePath().toString())
77+
.replace("TIKA_CONFIG", ASYNC_CONFIG.toAbsolutePath().toString());
78+
79+
;
80+
Files.writeString(ASYNC_PLUGINS_CONFIG, json, UTF_8);
6381
}
6482

6583
/**
@@ -103,19 +121,22 @@ private void resetContent() throws Exception {
103121

104122
@Test
105123
public void testAsync() throws Exception {
106-
String content = getParamOutContent("-a", "-c", ASYNC_CONFIG.toAbsolutePath().toString());
124+
//extension is "jsn" to avoid conflict with json config
125+
126+
String content = getParamOutContent("-c", ASYNC_CONFIG.toAbsolutePath().toString(),
127+
"-a", ASYNC_PLUGINS_CONFIG.toAbsolutePath().toString());
107128

108129
int json = 0;
109130
for (File f : ASYNC_OUTPUT_DIR
110131
.toFile()
111132
.listFiles()) {
112133
if (f
113134
.getName()
114-
.endsWith(".json")) {
135+
.endsWith(".jsn")) {
115136
//check one file for pretty print
116137
if (f
117138
.getName()
118-
.equals("coffee.xls.json")) {
139+
.equals("coffee.xls.jsn")) {
119140
checkForPrettyPrint(f);
120141
}
121142
json++;

‎tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,8 @@ public void testMacros() throws Exception {
281281

282282
@Test
283283
public void testRUnpack() throws Exception {
284+
//TODO -- rework this to use two separate emitters
285+
//one for bytes and one for json
284286
String[] expectedChildren = new String[]{
285287
"testPDFPackage.pdf.json",
286288
//the first two test that the default single file config is working
@@ -396,10 +398,17 @@ private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFil
396398

397399
private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFileNames, int expectedLength) throws Exception {
398400
Path input = Paths.get(new URI(resourcePrefix + "/" + targetFile));
399-
String[] params = {"-Z", input.toAbsolutePath().toString(),
400-
extractDir.toAbsolutePath().toString()};
401+
Path pluginsDir = Paths.get("target/plugins");
402+
403+
String[] params = {"-Z",
404+
"-p", ProcessUtils.escapeCommandLine(pluginsDir.toAbsolutePath().toString()),
405+
ProcessUtils.escapeCommandLine(input.toAbsolutePath().toString()),
406+
ProcessUtils.escapeCommandLine(extractDir
407+
.toAbsolutePath()
408+
.toString())};
401409

402410
TikaCLI.main(params);
411+
403412
Set<String> fileNames = getFileNames(extractDir);
404413
String[] jsonFile = extractDir
405414
.toFile()
@@ -408,7 +417,7 @@ private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFil
408417
assertEquals(expectedLength, jsonFile.length);
409418

410419
for (String expectedChildName : expectedChildrenFileNames) {
411-
assertTrue(fileNames.contains(expectedChildName));
420+
assertTrue(fileNames.contains(expectedChildName), expectedChildName);
412421
}
413422
}
414423

0 commit comments

Comments
 (0)