Skip to content

Commit 063cc8e

Browse files
tballisontballison
authored
TIKA-4553 -- rm TikaConfig from tika-detectors and tika-eval-core (#2434)
1 parent 56397f1 commit 063cc8e

11 files changed

Lines changed: 168 additions & 161 deletions

File tree

‎tika-detectors/tika-detector-magika/pom.xml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,20 @@
3939
<groupId>com.fasterxml.jackson.core</groupId>
4040
<artifactId>jackson-databind</artifactId>
4141
</dependency>
42+
<!-- Annotation processor - contains @TikaComponent and ensures build order.
43+
"provided" because it is only used at compile time -->
44+
<dependency>
45+
<groupId>org.apache.tika</groupId>
46+
<artifactId>tika-annotation-processor</artifactId>
47+
<version>${project.version}</version>
48+
<scope>provided</scope>
49+
</dependency>
50+
<dependency>
51+
<groupId>${project.groupId}</groupId>
52+
<artifactId>tika-serialization</artifactId>
53+
<version>${project.version}</version>
54+
<scope>test</scope>
55+
</dependency>
4256
<dependency>
4357
<groupId>org.apache.logging.log4j</groupId>
4458
<artifactId>log4j-core</artifactId>
@@ -68,6 +82,19 @@
6882
</excludes>
6983
</configuration>
7084
</plugin>
85+
<plugin>
86+
<groupId>org.apache.maven.plugins</groupId>
87+
<artifactId>maven-compiler-plugin</artifactId>
88+
<configuration>
89+
<annotationProcessorPaths>
90+
<path>
91+
<groupId>org.apache.tika</groupId>
92+
<artifactId>tika-annotation-processor</artifactId>
93+
<version>${project.version}</version>
94+
</path>
95+
</annotationProcessorPaths>
96+
</configuration>
97+
</plugin>
7198
<plugin>
7299
<groupId>org.apache.maven.plugins</groupId>
73100
<artifactId>maven-jar-plugin</artifactId>

‎tika-detectors/tika-detector-magika/src/main/java/org/apache/tika/detect/magika/MagikaDetector.java‎

Lines changed: 45 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,10 @@
3131
import org.slf4j.Logger;
3232
import org.slf4j.LoggerFactory;
3333

34+
import org.apache.tika.config.ConfigDeserializer;
3435
import org.apache.tika.config.Field;
36+
import org.apache.tika.config.JsonConfig;
37+
import org.apache.tika.config.TikaComponent;
3538
import org.apache.tika.detect.Detector;
3639
import org.apache.tika.io.BoundedInputStream;
3740
import org.apache.tika.io.TemporaryResources;
@@ -50,6 +53,7 @@
5053
* The default behavior is to run detection, report the results in the
5154
* metadata and then return null so that other detectors will be used.
5255
*/
56+
@TikaComponent
5357
public class MagikaDetector implements Detector {
5458

5559
enum STATUS {
@@ -90,11 +94,35 @@ enum STATUS {
9094
private static ObjectMapper OBJECT_MAPPER = new ObjectMapper();
9195
private static boolean HAS_WARNED = false;
9296
private Boolean hasMagika = null;
93-
private String magikaPath = DEFAULT_MAGIKA_PATH;
94-
private int maxBytes = 1_000_000;
95-
private long timeoutMs = DEFAULT_TIMEOUT_MS;
9697

97-
private boolean useMime = false;
98+
/**
99+
* Configuration class for JSON deserialization.
100+
*/
101+
public static class Config {
102+
public String magikaPath = DEFAULT_MAGIKA_PATH;
103+
public int maxBytes = 1_000_000;
104+
public long timeoutMs = DEFAULT_TIMEOUT_MS;
105+
public boolean useMime = false;
106+
}
107+
108+
private final Config config;
109+
110+
/**
111+
* Default constructor.
112+
*/
113+
public MagikaDetector() {
114+
this.config = new Config();
115+
}
116+
117+
/**
118+
* Constructor for JSON configuration.
119+
* Requires tika-serialization on the classpath.
120+
*
121+
* @param jsonConfig JSON configuration
122+
*/
123+
public MagikaDetector(JsonConfig jsonConfig) {
124+
this.config = ConfigDeserializer.buildConfig(jsonConfig, Config.class);
125+
}
98126

99127
public static boolean checkHasMagika(String magikaCommandPath) {
100128
String[] commandline = new String[]{magikaCommandPath, "--version"};
@@ -136,11 +164,11 @@ public static boolean checkHasMagika(String magikaCommandPath) {
136164
@Override
137165
public MediaType detect(InputStream input, Metadata metadata) throws IOException {
138166
if (hasMagika == null) {
139-
hasMagika = checkHasMagika(this.magikaPath);
167+
hasMagika = checkHasMagika(this.config.magikaPath);
140168
}
141169
if (!hasMagika) {
142170
if (!HAS_WARNED) {
143-
LOGGER.warn("'magika' command isn't working: '" + magikaPath + "'");
171+
LOGGER.warn("'magika' command isn't working: '" + config.magikaPath + "'");
144172
HAS_WARNED = true;
145173
}
146174
return MediaType.OCTET_STREAM;
@@ -152,10 +180,10 @@ public MediaType detect(InputStream input, Metadata metadata) throws IOException
152180
return detectOnPath(tis.getPath(), metadata);
153181
}
154182

155-
input.mark(maxBytes);
183+
input.mark(config.maxBytes);
156184
try (TemporaryResources tmp = new TemporaryResources()) {
157185
Path tmpFile = tmp.createTempFile();
158-
Files.copy(new BoundedInputStream(maxBytes, input), tmpFile, REPLACE_EXISTING);
186+
Files.copy(new BoundedInputStream(config.maxBytes, input), tmpFile, REPLACE_EXISTING);
159187
return detectOnPath(tmpFile, metadata);
160188
} finally {
161189
input.reset();
@@ -174,23 +202,23 @@ public MediaType detect(InputStream input, Metadata metadata) throws IOException
174202
*/
175203
@Field
176204
public void setUseMime(boolean useMime) {
177-
this.useMime = useMime;
205+
this.config.useMime = useMime;
178206
}
179207

180208
public boolean isUseMime() {
181-
return useMime;
209+
return config.useMime;
182210
}
183211

184212
private MediaType detectOnPath(Path path, Metadata metadata) throws IOException {
185213

186214
String[] args = new String[]{
187-
ProcessUtils.escapeCommandLine(magikaPath),
215+
ProcessUtils.escapeCommandLine(config.magikaPath),
188216
ProcessUtils.escapeCommandLine(path.toAbsolutePath().toString()),
189217
"--json"
190218
};
191219
ProcessBuilder builder = new ProcessBuilder(args);
192-
FileProcessResult result = ProcessUtils.execute(builder, timeoutMs, 10000000, 1000);
193-
return processResult(result, metadata, useMime);
220+
FileProcessResult result = ProcessUtils.execute(builder, config.timeoutMs, 10000000, 1000);
221+
return processResult(result, metadata, config.useMime);
194222
}
195223

196224
protected static MediaType processResult(FileProcessResult result, Metadata metadata,
@@ -331,8 +359,8 @@ private static void addString(JsonNode node, String jsonKey, Property property,
331359
public void setMagikaPath(String fileCommandPath) {
332360
//this opens up a potential command vulnerability.
333361
//Don't ever let an untrusted user set this.
334-
this.magikaPath = fileCommandPath;
335-
checkHasMagika(this.magikaPath);
362+
this.config.magikaPath = fileCommandPath;
363+
checkHasMagika(this.config.magikaPath);
336364
}
337365

338366
/**
@@ -344,11 +372,11 @@ public void setMagikaPath(String fileCommandPath) {
344372
*/
345373
@Field
346374
public void setMaxBytes(int maxBytes) {
347-
this.maxBytes = maxBytes;
375+
this.config.maxBytes = maxBytes;
348376
}
349377

350378
@Field
351379
public void setTimeoutMs(long timeoutMs) {
352-
this.timeoutMs = timeoutMs;
380+
this.config.timeoutMs = timeoutMs;
353381
}
354382
}

‎tika-detectors/tika-detector-magika/src/test/java/org/apache/tika/detect/magika/TestMagikaIntegration.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,18 +27,17 @@
2727
import org.junit.jupiter.api.Test;
2828

2929
import org.apache.tika.TikaTest;
30-
import org.apache.tika.config.TikaConfig;
30+
import org.apache.tika.config.loader.TikaLoader;
3131
import org.apache.tika.metadata.Metadata;
32-
import org.apache.tika.parser.AutoDetectParser;
3332
import org.apache.tika.parser.Parser;
3433

3534
@Disabled("need to have magika on the path")
3635
public class TestMagikaIntegration extends TikaTest {
3736

3837
@Test
3938
public void testIntegration() throws Exception {
40-
TikaConfig tikaConfig = new TikaConfig(getConfig("tika-config.xml"));
41-
Parser p = new AutoDetectParser(tikaConfig);
39+
TikaLoader tikaLoader = TikaLoader.load(getConfig("tika-config.json"));
40+
Parser p = tikaLoader.loadAutoDetectParser();
4241
List<Metadata> metadataList = getRecursiveMetadata("testPDF.pdf", p);
4342
debug(getRecursiveMetadata("testPDF.pdf", p));
4443
Metadata m = metadataList.get(0);

‎tika-detectors/tika-detector-magika/src/test/resources/configs/tika-config.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"detectors": [
3+
{
4+
"default-detector": {}
5+
},
6+
{
7+
"magika-detector": {
8+
"useMime": true
9+
}
10+
}
11+
]
12+
}

‎tika-detectors/tika-detector-magika/src/test/resources/configs/tika-config.xml‎

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎tika-detectors/tika-detector-siegfried/pom.xml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,14 @@
3939
<groupId>com.fasterxml.jackson.core</groupId>
4040
<artifactId>jackson-databind</artifactId>
4141
</dependency>
42+
<!-- Annotation processor - contains @TikaComponent and ensures build order.
43+
"provided" because it is only used at compile time -->
44+
<dependency>
45+
<groupId>org.apache.tika</groupId>
46+
<artifactId>tika-annotation-processor</artifactId>
47+
<version>${project.version}</version>
48+
<scope>provided</scope>
49+
</dependency>
4250
<!-- test dependencies -->
4351
<dependency>
4452
<groupId>${project.groupId}</groupId>
@@ -75,6 +83,19 @@
7583
</excludes>
7684
</configuration>
7785
</plugin>
86+
<plugin>
87+
<groupId>org.apache.maven.plugins</groupId>
88+
<artifactId>maven-compiler-plugin</artifactId>
89+
<configuration>
90+
<annotationProcessorPaths>
91+
<path>
92+
<groupId>org.apache.tika</groupId>
93+
<artifactId>tika-annotation-processor</artifactId>
94+
<version>${project.version}</version>
95+
</path>
96+
</annotationProcessorPaths>
97+
</configuration>
98+
</plugin>
7899
<plugin>
79100
<groupId>org.apache.maven.plugins</groupId>
80101
<artifactId>maven-jar-plugin</artifactId>

0 commit comments

Comments
 (0)