Skip to content

Commit 3ae604c

Browse files
tballisontballison
authored
TIKA-4769 - improve metadata robustness (#2923)
1 parent 6b8a950 commit 3ae604c

49 files changed

Lines changed: 422 additions & 136 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎tika-core/src/main/java/org/apache/tika/digest/InputStreamDigester.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,12 @@
2020
import java.security.MessageDigest;
2121
import java.security.NoSuchAlgorithmException;
2222
import java.security.Provider;
23+
import java.util.Map;
24+
import java.util.concurrent.ConcurrentHashMap;
2325

2426
import org.apache.tika.io.TikaInputStream;
2527
import org.apache.tika.metadata.Metadata;
28+
import org.apache.tika.metadata.Property;
2629
import org.apache.tika.parser.ParseContext;
2730
import org.apache.tika.utils.StringUtils;
2831

@@ -32,8 +35,10 @@
3235
*/
3336
public class InputStreamDigester implements Digester {
3437

38+
private static final Map<String, Property> METADATA_KEY_PROPERTIES = new ConcurrentHashMap<>();
39+
3540
private final String algorithm;
36-
private final String metadataKey;
41+
private final Property metadataProperty;
3742
private final Encoder encoder;
3843

3944
/**
@@ -45,7 +50,8 @@ public class InputStreamDigester implements Digester {
4550
*/
4651
public InputStreamDigester(String algorithm, String metadataKey, Encoder encoder) {
4752
this.algorithm = algorithm;
48-
this.metadataKey = metadataKey;
53+
this.metadataProperty =
54+
METADATA_KEY_PROPERTIES.computeIfAbsent(metadataKey, Property::internalText);
4955
this.encoder = encoder;
5056
}
5157

@@ -107,7 +113,7 @@ public void digest(TikaInputStream tis, Metadata metadata, ParseContext parseCon
107113
}
108114

109115
setContentLength(total, metadata);
110-
metadata.set(metadataKey, encoder.encode(messageDigest.digest()));
116+
metadata.set(metadataProperty, encoder.encode(messageDigest.digest()));
111117

112118
tis.rewind();
113119
}

‎tika-core/src/main/java/org/apache/tika/metadata/Metadata.java‎

Lines changed: 74 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@
3434
import java.util.Properties;
3535
import java.util.TimeZone;
3636

37+
import org.slf4j.Logger;
38+
import org.slf4j.LoggerFactory;
39+
3740
import org.apache.tika.metadata.Property.PropertyType;
3841
import org.apache.tika.metadata.writefilter.MetadataWriteLimiter;
3942
import org.apache.tika.metadata.writefilter.MetadataWriteLimiterFactory;
@@ -47,6 +50,7 @@ public class Metadata
4750
implements CreativeCommons, Geographic, HttpHeaders, Message, ClimateForcast, TIFF,
4851
TikaMimeKeys, Serializable {
4952

53+
private static final Logger LOG = LoggerFactory.getLogger(Metadata.class);
5054

5155
private static final MetadataWriteLimiter ACCEPT_ALL = new MetadataWriteLimiter() {
5256
@Override
@@ -96,6 +100,7 @@ private String[] appendValues(String[] values, final String value) {
96100

97101

98102
private MetadataWriteLimiter writeLimiter = ACCEPT_ALL;
103+
private transient boolean trusted;
99104
/**
100105
* Constructs a new, empty metadata.
101106
*/
@@ -304,9 +309,66 @@ private String[] _getValues(final String name) {
304309
* @param value the metadata value.
305310
*/
306311
public void add(final String name, final String value) {
312+
if (blockReservedKeyWrite(name)) {
313+
return;
314+
}
315+
addUnchecked(name, value);
316+
}
317+
318+
/** Trusted add, bypassing the reserved-key guard. */
319+
private void addUnchecked(final String name, final String value) {
307320
writeLimiter.add(name, value, metadata);
308321
}
309322

323+
/**
324+
* Mark this Metadata as a trusted transformation target (e.g. a metadata filter), letting
325+
* String writes reach reserved {@code X-TIKA:} keys. Reset when the transformation is done.
326+
*/
327+
public void setTrusted(boolean trusted) {
328+
this.trusted = trusted;
329+
}
330+
331+
public boolean isTrusted() {
332+
return trusted;
333+
}
334+
335+
/** Drop String writes to reserved {@code X-TIKA:} keys unless trusted; use their Property. */
336+
private boolean blockReservedKeyWrite(String name) {
337+
if (!trusted && name != null && name.startsWith(TikaCoreProperties.TIKA_META_PREFIX)) {
338+
LOG.debug("Dropping String write to reserved metadata key '{}'; use its Property.", name);
339+
return true;
340+
}
341+
return false;
342+
}
343+
344+
/**
345+
* Trusted write for clone/merge/deserialize; reserved keys go via their Property.
346+
*
347+
* @param append add rather than set
348+
*/
349+
public void reconstruct(String name, String value, boolean append) {
350+
if (name != null && name.startsWith(TikaCoreProperties.TIKA_META_PREFIX)) {
351+
Property property = Property.get(name);
352+
if (property != null) {
353+
if (append) {
354+
add(property, value);
355+
} else {
356+
set(property, value);
357+
}
358+
} else if (append) {
359+
addUnchecked(name, value);
360+
} else {
361+
setUnchecked(name, value);
362+
}
363+
return;
364+
}
365+
if (append) {
366+
add(name, value);
367+
} else {
368+
set(name, value);
369+
}
370+
}
371+
310372
/**
311373
* Add a metadata name/value mapping. Add the specified value to the list of
312374
* values associated to the specified metadata name.
@@ -320,7 +382,7 @@ protected void add(final String name, final String[] newValues) {
320382
set(name, newValues);
321383
} else {
322384
for (String val : newValues) {
323-
add(name, val);
385+
addUnchecked(name, val);
324386
}
325387
}
326388
}
@@ -351,7 +413,7 @@ public void add(final Property property, final String value) {
351413
set(property, value);
352414
} else {
353415
if (property.isMultiValuePermitted()) {
354-
add(property.getName(), value);
416+
addUnchecked(property.getName(), value);
355417
} else {
356418
throw new PropertyTypeException(
357419
property.getName() + " : " + property.getPropertyType());
@@ -384,6 +446,14 @@ public void setAll(Properties properties) {
384446
* @param value the metadata value, or <code>null</code>
385447
*/
386448
public void set(String name, String value) {
449+
if (blockReservedKeyWrite(name)) {
450+
return;
451+
}
452+
setUnchecked(name, value);
453+
}
454+
455+
/** Trusted set, bypassing the reserved-key guard. */
456+
private void setUnchecked(String name, String value) {
387457
writeLimiter.set(name, value, metadata);
388458
}
389459

@@ -393,7 +463,7 @@ protected void set(String name, String[] values) {
393463
if (values != null) {
394464
metadata.remove(name);
395465
for (String v : values) {
396-
add(name, v);
466+
addUnchecked(name, v);
397467
}
398468
} else {
399469
metadata.remove(name);
@@ -419,7 +489,7 @@ public void set(Property property, String value) {
419489
}
420490
}
421491
} else {
422-
set(property.getName(), value);
492+
setUnchecked(property.getName(), value);
423493
}
424494
}
425495

‎tika-core/src/main/java/org/apache/tika/metadata/filter/CompositeMetadataFilter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ public List<MetadataFilter> getFilters() {
4646
}
4747

4848
@Override
49-
public void filter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
49+
protected void doFilter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
5050
for (MetadataFilter filter : filters) {
5151
filter.filter(metadataList, parseContext);
5252
}

‎tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilter.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@
1919
import java.io.Closeable;
2020
import java.io.IOException;
2121
import java.io.Serializable;
22+
import java.util.IdentityHashMap;
2223
import java.util.List;
24+
import java.util.Map;
2325

2426
import org.apache.tika.exception.TikaException;
2527
import org.apache.tika.metadata.Metadata;
@@ -37,7 +39,29 @@ public abstract class MetadataFilter implements Serializable, Closeable {
3739
* @param parseContext per-request context (e.g. skip flags, runtime config)
3840
* @throws TikaException if filtering fails
3941
*/
40-
public abstract void filter(List<Metadata> metadataList, ParseContext parseContext)
42+
public final void filter(List<Metadata> metadataList, ParseContext parseContext)
43+
throws TikaException {
44+
Map<Metadata, Boolean> previous = new IdentityHashMap<>();
45+
for (Metadata m : metadataList) {
46+
previous.put(m, m.isTrusted());
47+
m.setTrusted(true);
48+
}
49+
try {
50+
doFilter(metadataList, parseContext);
51+
} finally {
52+
previous.forEach(Metadata::setTrusted);
53+
}
54+
}
55+
56+
/**
57+
* Applies the filter in place. Reserved-key String writes are permitted here: filters run
58+
* on already-parsed, trusted metadata.
59+
*
60+
* @param metadataList the list to filter (must be mutable)
61+
* @param parseContext per-request context
62+
* @throws TikaException if filtering fails
63+
*/
64+
protected abstract void doFilter(List<Metadata> metadataList, ParseContext parseContext)
4165
throws TikaException;
4266

4367
/**

‎tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilterBase.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@
3030
public abstract class MetadataFilterBase extends MetadataFilter {
3131

3232
@Override
33-
public void filter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
33+
protected void doFilter(List<Metadata> metadataList, ParseContext parseContext)
34+
throws TikaException {
3435
for (Metadata m : metadataList) {
3536
filter(m);
3637
}

‎tika-core/src/main/java/org/apache/tika/metadata/filter/NoOpFilter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public class NoOpFilter extends MetadataFilter {
3434

3535

3636
@Override
37-
public void filter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
37+
protected void doFilter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
3838
// no-op
3939
}
4040
}

‎tika-core/src/main/java/org/apache/tika/metadata/filter/RemoveByMimeMetadataFilter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ public RemoveByMimeMetadataFilter(JsonConfig jsonConfig) {
7474
}
7575

7676
@Override
77-
public void filter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
77+
protected void doFilter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
7878
metadataList.removeIf(this::shouldRemove);
7979
}
8080

‎tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -345,7 +345,7 @@ private void recordEmbeddedMetadata(Metadata metadata, ParseContext context) {
345345
for (Metadata m : record.getMetadataList()) {
346346
for (String n : m.names()) {
347347
for (String v : m.getValues(n)) {
348-
metadata.add(n, v);
348+
metadata.reconstruct(n, v, true);
349349
}
350350
}
351351
}

‎tika-core/src/main/java/org/apache/tika/parser/multiple/AbstractMultipleParser.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ protected static Metadata mergeMetadata(Metadata newMetadata, Metadata lastMetad
125125
if (newVals == null || newVals.length == 0) {
126126
// Metadata only in previous run, keep old values
127127
for (String val : oldVals) {
128-
newMetadata.add(n, val);
128+
newMetadata.reconstruct(n, val, true);
129129
}
130130
} else if (Arrays.deepEquals(oldVals, newVals)) {
131131
// Metadata is the same, nothing to do
@@ -136,7 +136,7 @@ protected static Metadata mergeMetadata(Metadata newMetadata, Metadata lastMetad
136136
// Use the earlier value(s) in place of this/these one/s
137137
newMetadata.remove(n);
138138
for (String val : oldVals) {
139-
newMetadata.add(n, val);
139+
newMetadata.reconstruct(n, val, true);
140140
}
141141
continue;
142142
case LAST_WINS:
@@ -147,11 +147,11 @@ protected static Metadata mergeMetadata(Metadata newMetadata, Metadata lastMetad
147147
List<String> vals = new ArrayList<>(Arrays.asList(oldVals));
148148
newMetadata.remove(n);
149149
for (String oldVal : oldVals) {
150-
newMetadata.add(n, oldVal);
150+
newMetadata.reconstruct(n, oldVal, true);
151151
}
152152
for (String newVal : newVals) {
153153
if (!vals.contains(newVal)) {
154-
newMetadata.add(n, newVal);
154+
newMetadata.reconstruct(n, newVal, true);
155155
vals.add(newVal);
156156
}
157157
}
@@ -318,7 +318,7 @@ private void parse(TikaInputStream tis, ContentHandler handler,
318318
for (String n : metadata.names()) {
319319
originalMetadata.remove(n);
320320
for (String val : metadata.getValues(n)) {
321-
originalMetadata.add(n, val);
321+
originalMetadata.reconstruct(n, val, true);
322322
}
323323
}
324324
}

‎tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,10 @@ public static Metadata cloneMetadata(Metadata m) {
4343

4444
for (String n : m.names()) {
4545
if (!m.isMultiValued(n)) {
46-
clone.set(n, m.get(n));
46+
clone.reconstruct(n, m.get(n), false);
4747
} else {
48-
String[] vals = m.getValues(n);
49-
for (String val : vals) {
50-
clone.add(n, val);
48+
for (String val : m.getValues(n)) {
49+
clone.reconstruct(n, val, true);
5150
}
5251
}
5352
}

0 commit comments

Comments
 (0)