TIKA-4769 - improve metadata robustness

tika-core/src/main/java/org/apache/tika/digest/InputStreamDigester.java

@@ -20,9 +20,12 @@
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.security.Provider;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
 import org.apache.tika.parser.ParseContext;
 import org.apache.tika.utils.StringUtils;
 
@@ -32,8 +35,10 @@
  */
 public class InputStreamDigester implements Digester {
 
+    private static final Map<String, Property> METADATA_KEY_PROPERTIES = new ConcurrentHashMap<>();
+
     private final String algorithm;
-    private final String metadataKey;
+    private final Property metadataProperty;
     private final Encoder encoder;
 
     /**
@@ -45,7 +50,8 @@ public class InputStreamDigester implements Digester {
      */
     public InputStreamDigester(String algorithm, String metadataKey, Encoder encoder) {
         this.algorithm = algorithm;
-        this.metadataKey = metadataKey;
+        this.metadataProperty =
+                METADATA_KEY_PROPERTIES.computeIfAbsent(metadataKey, Property::internalText);
         this.encoder = encoder;
     }
 
@@ -107,7 +113,7 @@ public void digest(TikaInputStream tis, Metadata metadata, ParseContext parseCon
         }
 
         setContentLength(total, metadata);
-        metadata.set(metadataKey, encoder.encode(messageDigest.digest()));
+        metadata.set(metadataProperty, encoder.encode(messageDigest.digest()));
 
         tis.rewind();
     }

tika-core/src/main/java/org/apache/tika/metadata/Metadata.java

@@ -34,6 +34,9 @@
 import java.util.Properties;
 import java.util.TimeZone;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.tika.metadata.Property.PropertyType;
 import org.apache.tika.metadata.writefilter.MetadataWriteLimiter;
 import org.apache.tika.metadata.writefilter.MetadataWriteLimiterFactory;
@@ -47,6 +50,7 @@ public class Metadata
         implements CreativeCommons, Geographic, HttpHeaders, Message, ClimateForcast, TIFF,
         TikaMimeKeys, Serializable {
 
+    private static final Logger LOG = LoggerFactory.getLogger(Metadata.class);
 
     private static final MetadataWriteLimiter ACCEPT_ALL = new MetadataWriteLimiter() {
         @Override
@@ -96,6 +100,7 @@ private String[] appendValues(String[] values, final String value) {
 
 
     private MetadataWriteLimiter writeLimiter = ACCEPT_ALL;
+    private transient boolean trusted;
     /**
      * Constructs a new, empty metadata.
      */
@@ -304,9 +309,66 @@ private String[] _getValues(final String name) {
      * @param value the metadata value.
      */
     public void add(final String name, final String value) {
+        if (blockReservedKeyWrite(name)) {
+            return;
+        }
+        addUnchecked(name, value);
+    }
+
+    /** Trusted add, bypassing the reserved-key guard. */
+    private void addUnchecked(final String name, final String value) {
         writeLimiter.add(name, value, metadata);
     }
 
+    /**
+     * Mark this Metadata as a trusted transformation target (e.g. a metadata filter), letting
+     * String writes reach reserved {@code X-TIKA:} keys. Reset when the transformation is done.
+     */
+    public void setTrusted(boolean trusted) {
+        this.trusted = trusted;
+    }
+
+    public boolean isTrusted() {
+        return trusted;
+    }
+
+    /** Drop String writes to reserved {@code X-TIKA:} keys unless trusted; use their Property. */
+    private boolean blockReservedKeyWrite(String name) {
+        if (!trusted && name != null && name.startsWith(TikaCoreProperties.TIKA_META_PREFIX)) {
+            LOG.debug("Dropping String write to reserved metadata key '{}'; use its Property.", name);
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Trusted write for clone/merge/deserialize; reserved keys go via their Property.
+     *
+     * @param append add rather than set
+     */
+    public void reconstruct(String name, String value, boolean append) {
+        if (name != null && name.startsWith(TikaCoreProperties.TIKA_META_PREFIX)) {
+            Property property = Property.get(name);
+            if (property != null) {
+                if (append) {
+                    add(property, value);
+                } else {
+                    set(property, value);
+                }
+            } else if (append) {
+                addUnchecked(name, value);
+            } else {
+                setUnchecked(name, value);
+            }
+            return;
+        }
+        if (append) {
+            add(name, value);
+        } else {
+            set(name, value);
+        }
+    }
+
     /**
      * Add a metadata name/value mapping. Add the specified value to the list of
      * values associated to the specified metadata name.
@@ -320,7 +382,7 @@ protected void add(final String name, final String[] newValues) {
             set(name, newValues);
         } else {
             for (String val : newValues) {
-                add(name, val);
+                addUnchecked(name, val);
             }
         }
     }
@@ -351,7 +413,7 @@ public void add(final Property property, final String value) {
                 set(property, value);
             } else {
                 if (property.isMultiValuePermitted()) {
-                    add(property.getName(), value);
+                    addUnchecked(property.getName(), value);
                 } else {
                     throw new PropertyTypeException(
                             property.getName() + " : " + property.getPropertyType());
@@ -384,6 +446,14 @@ public void setAll(Properties properties) {
      * @param value the metadata value, or <code>null</code>
      */
     public void set(String name, String value) {
+        if (blockReservedKeyWrite(name)) {
+            return;
+        }
+        setUnchecked(name, value);
+    }
+
+    /** Trusted set, bypassing the reserved-key guard. */
+    private void setUnchecked(String name, String value) {
         writeLimiter.set(name, value, metadata);
     }
 
@@ -393,7 +463,7 @@ protected void set(String name, String[] values) {
         if (values != null) {
             metadata.remove(name);
             for (String v : values) {
-                add(name, v);
+                addUnchecked(name, v);
             }
         } else {
             metadata.remove(name);
@@ -419,7 +489,7 @@ public void set(Property property, String value) {
                 }
             }
         } else {
-            set(property.getName(), value);
+            setUnchecked(property.getName(), value);
         }
     }
 

tika-core/src/main/java/org/apache/tika/metadata/filter/CompositeMetadataFilter.java

@@ -46,7 +46,7 @@ public List<MetadataFilter> getFilters() {
     }
 
     @Override
-    public void filter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
+    protected void doFilter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
         for (MetadataFilter filter : filters) {
             filter.filter(metadataList, parseContext);
         }

tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilter.java

@@ -19,7 +19,9 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.tika.exception.TikaException;
 import org.apache.tika.metadata.Metadata;
@@ -37,7 +39,29 @@ public abstract class MetadataFilter implements Serializable, Closeable {
      * @param parseContext per-request context (e.g. skip flags, runtime config)
      * @throws TikaException if filtering fails
      */
-    public abstract void filter(List<Metadata> metadataList, ParseContext parseContext)
+    public final void filter(List<Metadata> metadataList, ParseContext parseContext)
+            throws TikaException {
+        Map<Metadata, Boolean> previous = new IdentityHashMap<>();
+        for (Metadata m : metadataList) {
+            previous.put(m, m.isTrusted());
+            m.setTrusted(true);
+        }
+        try {
+            doFilter(metadataList, parseContext);
+        } finally {
+            previous.forEach(Metadata::setTrusted);
+        }
+    }
+
+    /**
+     * Applies the filter in place. Reserved-key String writes are permitted here: filters run
+     * on already-parsed, trusted metadata.
+     *
+     * @param metadataList the list to filter (must be mutable)
+     * @param parseContext per-request context
+     * @throws TikaException if filtering fails
+     */
+    protected abstract void doFilter(List<Metadata> metadataList, ParseContext parseContext)
             throws TikaException;
 
     /**

tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilterBase.java

@@ -30,7 +30,8 @@
 public abstract class MetadataFilterBase extends MetadataFilter {
 
     @Override
-    public void filter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
+    protected void doFilter(List<Metadata> metadataList, ParseContext parseContext)
+            throws TikaException {
         for (Metadata m : metadataList) {
             filter(m);
         }

tika-core/src/main/java/org/apache/tika/metadata/filter/NoOpFilter.java

@@ -34,7 +34,7 @@ public class NoOpFilter extends MetadataFilter {
 
 
     @Override
-    public void filter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
+    protected void doFilter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
         // no-op
     }
 }

tika-core/src/main/java/org/apache/tika/metadata/filter/RemoveByMimeMetadataFilter.java

@@ -74,7 +74,7 @@ public RemoveByMimeMetadataFilter(JsonConfig jsonConfig) {
     }
 
     @Override
-    public void filter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
+    protected void doFilter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
         metadataList.removeIf(this::shouldRemove);
     }
 

tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java

@@ -345,7 +345,7 @@ private void recordEmbeddedMetadata(Metadata metadata, ParseContext context) {
         for (Metadata m : record.getMetadataList()) {
             for (String n : m.names()) {
                 for (String v : m.getValues(n)) {
-                    metadata.add(n, v);
+                    metadata.reconstruct(n, v, true);
                 }
             }
         }

tika-core/src/main/java/org/apache/tika/parser/multiple/AbstractMultipleParser.java

@@ -125,7 +125,7 @@ protected static Metadata mergeMetadata(Metadata newMetadata, Metadata lastMetad
             if (newVals == null || newVals.length == 0) {
                 // Metadata only in previous run, keep old values
                 for (String val : oldVals) {
-                    newMetadata.add(n, val);
+                    newMetadata.reconstruct(n, val, true);
                 }
             } else if (Arrays.deepEquals(oldVals, newVals)) {
                 // Metadata is the same, nothing to do
@@ -136,7 +136,7 @@ protected static Metadata mergeMetadata(Metadata newMetadata, Metadata lastMetad
                         // Use the earlier value(s) in place of this/these one/s
                         newMetadata.remove(n);
                         for (String val : oldVals) {
-                            newMetadata.add(n, val);
+                            newMetadata.reconstruct(n, val, true);
                         }
                         continue;
                     case LAST_WINS:
@@ -147,11 +147,11 @@ protected static Metadata mergeMetadata(Metadata newMetadata, Metadata lastMetad
                         List<String> vals = new ArrayList<>(Arrays.asList(oldVals));
                         newMetadata.remove(n);
                         for (String oldVal : oldVals) {
-                            newMetadata.add(n, oldVal);
+                            newMetadata.reconstruct(n, oldVal, true);
                         }
                         for (String newVal : newVals) {
                             if (!vals.contains(newVal)) {
-                                newMetadata.add(n, newVal);
+                                newMetadata.reconstruct(n, newVal, true);
                                 vals.add(newVal);
                             }
                         }
@@ -318,7 +318,7 @@ private void parse(TikaInputStream tis, ContentHandler handler,
         for (String n : metadata.names()) {
             originalMetadata.remove(n);
             for (String val : metadata.getValues(n)) {
-                originalMetadata.add(n, val);
+                originalMetadata.reconstruct(n, val, true);
             }
         }
     }

tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java

@@ -43,11 +43,10 @@ public static Metadata cloneMetadata(Metadata m) {
 
         for (String n : m.names()) {
             if (!m.isMultiValued(n)) {
-                clone.set(n, m.get(n));
+                clone.reconstruct(n, m.get(n), false);
             } else {
-                String[] vals = m.getValues(n);
-                for (String val : vals) {
-                    clone.add(n, val);
+                for (String val : m.getValues(n)) {
+                    clone.reconstruct(n, val, true);
                 }
             }
         }