TIKA-4797 add migration table (#2978)

Author: tballison

docs/modules/ROOT/pages/migration-to-4x/metadata-changes-4x.adoc

@@ -307,6 +307,13 @@ String value = metadata.get("mapi:some-property");
 String value = metadata.get("mapi:some-property");
 ----
 
+[CAUTION]
+====
+`pst:folderPath` does not migrate one-to-one, and the opt-in `LegacyKeyMigrationFilter` cannot
+restore it: it is *dropped*, not renamed. In 4.x the Outlook PST parser folds the folder path into
+`tk:internal-path` (`<folder>/<item-name>`, a superset), so there is no folder-only value to map back to.
+====
+
 === Resource Name
 
 [source,java]

tika-core/pom.xml

@@ -207,6 +207,7 @@
             <inputExclude>src/test/resources/test-documents/file-list.txt</inputExclude>
             <inputExclude>src/test/resources/test-documents/ang20150420t182050_corr_v1e_img.hdr</inputExclude>
             <inputExclude>src/test/resources/test-documents/*.pdf</inputExclude>
+            <inputExclude>src/main/resources/org/apache/tika/metadata/metadata-migration-3x-4x.json</inputExclude>
           </inputExcludes>
         </configuration>
       </plugin>

tika-core/src/main/java/org/apache/tika/metadata/MAPI.java

@@ -49,6 +49,9 @@ public interface MAPI {
 
     Property SUBMISSION_ACCEPTED_AT_TIME = Property.internalDate(PREFIX_MAPI_META + "msg-submission-accepted-at-time");
 
+    // PidTagClientSubmitTime -- distinct MAPI prop from the provider-side SUBMISSION_ACCEPTED_AT_TIME.
+    Property CLIENT_SUBMIT_TIME = Property.internalDate(PREFIX_MAPI_META + "msg-client-submit-time");
+
     Property SUBMISSION_ID = Property.internalText(PREFIX_MAPI_META + "msg-submission-id");
 
     Property INTERNET_MESSAGE_ID = Property.internalText(PREFIX_MAPI_META + "internet-message-id");

tika-core/src/main/java/org/apache/tika/metadata/filter/LegacyKeyMigrationFilter.java

@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata.filter;
+
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.UnaryOperator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.tika.annotation.TikaComponent;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+
+/**
+ * Rewrites 4.x metadata keys back to their 3.x spellings (or vice versa) so consumers pinned to 3.x
+ * key names keep working after the 4.0 rename break. Opt-in; enable it in the metadata-filter chain
+ * and it rewrites at the emit edge. Because it runs inside the {@link MetadataFilter} trusted bracket,
+ * it may write reserved (e.g. {@code X-TIKA:}/{@code tk:}) keys that a plain String write couldn't.
+ *
+ * <p><b>Data-driven, not hand-coded.</b> The renames come from the committed migration table
+ * (generated by joining the 3.x and 4.x field-attributed key tables — TIKA-4797). Three tiers:
+ * <ul>
+ *   <li><b>flat renames</b> — the enumerated key-for-key changes (the bulk);</li>
+ *   <li><b>drops</b> — 3.x keys 4.x no longer emits (only relevant on ingest);</li>
+ *   <li><b>prefix rules</b> — the open families whose suffix also transforms
+ *       ({@code tk:exception:}/{@code tk:warn:} snake&harr;kebab, {@code tk:digest:} algorithm names).</li>
+ * </ul>
+ * Unmapped keys pass through unchanged (this is a compatibility bridge, not an allow-list).
+ *
+ * <p><b>Skeleton status (TIKA-4799):</b> the flat/drop/passthrough mechanism is complete; the
+ * bundled table and the concrete prefix rules land with the rename PRs (the table resource is absent
+ * until then, so the default filter is a safe no-op). See {@link #prefixRules} for where the
+ * digest/exception rules slot in.
+ */
+@TikaComponent
+public class LegacyKeyMigrationFilter extends MetadataFilterBase {
+
+    /** V4_TO_V3 = egress (4.x output rewritten to 3.x keys, the BWC case); V3_TO_V4 = ingest. */
+    public enum Direction { V4_TO_V3, V3_TO_V4 }
+
+    /** Generated by the TIKA-4797 migration join; deposited into tika-core resources when renames land. */
+    static final String DEFAULT_TABLE = "/org/apache/tika/metadata/metadata-migration-3x-4x.json";
+
+    // {"v3":"...","v4":"..."} ; v4 == "DROPPED" marks a 3.x key with no 4.x successor.
+    private static final Pattern ROW =
+            Pattern.compile("\\{\"v3\":\"(.*?)\",\"v4\":\"(.*?)\"}");
+
+    // 4.x digest keys use the JCA name (DigestDef.getJavaName); 3.x used the enum name. MD2/MD5 match.
+    private static final Map<String, String> DIGEST_ALG_V4_TO_V3 = Map.of(
+            "SHA-1", "SHA1", "SHA-256", "SHA256", "SHA-384", "SHA384", "SHA-512", "SHA512",
+            "SHA3-256", "SHA3_256", "SHA3-384", "SHA3_384", "SHA3-512", "SHA3_512");
+    private static final String V4_DIGEST = TikaCoreProperties.TIKA_META_PREFIX + "digest:";
+    private static final String V3_DIGEST = TikaCoreProperties.LEGACY_TIKA_META_PREFIX + "digest:";
+
+    public static class Config {
+        public Direction direction = Direction.V4_TO_V3;
+        public String table = DEFAULT_TABLE;
+    }
+
+    private final Map<String, String> renames;   // source key -> target key (direction applied)
+    private final Set<String> drops;             // source keys to remove (ingest only)
+    private final List<UnaryOperator<String>> prefixRules;
+
+    public LegacyKeyMigrationFilter() {
+        this(new Config());
+    }
+
+    public LegacyKeyMigrationFilter(Config config) {
+        this(loadTable(config.table), config.direction);
+    }
+
+    /** Test/explicit hook. {@code table} is v3-&gt;v4 (with {@code null}/"DROPPED" = dropped). */
+    LegacyKeyMigrationFilter(Map<String, String> table, Direction direction) {
+        this.renames = new HashMap<>();
+        this.drops = new HashSet<>();
+        for (Map.Entry<String, String> e : table.entrySet()) {
+            String v3 = e.getKey();
+            String v4 = e.getValue();
+            boolean dropped = v4 == null || "DROPPED".equals(v4);
+            if (direction == Direction.V4_TO_V3) {
+                if (!dropped) {
+                    renames.put(v4, v3);       // 4.x key -> 3.x key
+                }
+            } else {                            // V3_TO_V4
+                if (dropped) {
+                    drops.add(v3);
+                } else {
+                    renames.put(v3, v4);
+                }
+            }
+        }
+        // Digest keys have no declaring field, so they aren't in the flat table -- rewrite them by
+        // prefix rule instead. (Exception/warn keys ARE enumerated in the table.)
+        this.prefixRules = new ArrayList<>();
+        this.prefixRules.add(digestRule(direction));
+    }
+
+    @Override
+    protected void filter(Metadata metadata) {
+        for (String name : metadata.names()) {          // names() returns a snapshot copy
+            if (drops.contains(name)) {
+                metadata.remove(name);
+                continue;
+            }
+            String mapped = renames.get(name);
+            if (mapped == null) {
+                for (UnaryOperator<String> rule : prefixRules) {
+                    String r = rule.apply(name);
+                    if (r != null) {
+                        mapped = r;
+                        break;
+                    }
+                }
+            }
+            if (mapped != null && !mapped.equals(name)) {
+                String[] values = metadata.getValues(name);
+                metadata.remove(name);
+                for (String v : values) {
+                    metadata.add(mapped, v);            // trusted context: reserved keys allowed
+                }
+            }
+            // unmapped -> pass through unchanged
+        }
+    }
+
+    private static Map<String, String> loadTable(String resource) {
+        Map<String, String> table = new HashMap<>();
+        try (InputStream in = LegacyKeyMigrationFilter.class.getResourceAsStream(resource)) {
+            if (in == null) {
+                return table;   // not bundled yet -> no-op bridge
+            }
+            Matcher m = ROW.matcher(new String(in.readAllBytes(), StandardCharsets.UTF_8));
+            while (m.find()) {
+                table.put(unescape(m.group(1)), unescape(m.group(2)));
+            }
+        } catch (Exception e) {
+            throw new IllegalStateException("could not read migration table " + resource, e);
+        }
+        return table;
+    }
+
+    private static String unescape(String s) {
+        return s.replace("\\\"", "\"").replace("\\\\", "\\");
+    }
+
+    /** {@code tk:digest:<jca-alg>[:enc]} &harr; {@code X-TIKA:digest:<enum-alg>[:enc]} (encoding kept). */
+    private static UnaryOperator<String> digestRule(Direction direction) {
+        boolean egress = direction == Direction.V4_TO_V3;
+        String fromPrefix = egress ? V4_DIGEST : V3_DIGEST;
+        String toPrefix = egress ? V3_DIGEST : V4_DIGEST;
+        Map<String, String> alg = new HashMap<>();
+        DIGEST_ALG_V4_TO_V3.forEach((v4, v3) -> alg.put(egress ? v4 : v3, egress ? v3 : v4));
+        return name -> {
+            if (!name.startsWith(fromPrefix)) {
+                return null;
+            }
+            String rest = name.substring(fromPrefix.length());   // <alg> or <alg>:<enc>
+            int c = rest.indexOf(':');
+            String a = c < 0 ? rest : rest.substring(0, c);
+            String enc = c < 0 ? "" : rest.substring(c);
+            return toPrefix + alg.getOrDefault(a, a) + enc;
+        };
+    }
+}

tika-core/src/main/resources/org/apache/tika/metadata/metadata-migration-3x-4x.json

@@ -0,0 +1,137 @@
+[
+{"v3":"Content-Type-Hint","v4":"tk:content-type-hint"},
+{"v3":"Content-Type-Override","v4":"tk:content-type-override"},
+{"v3":"Content-Type-Parser-Override","v4":"tk:content-type-parser-override"},
+{"v3":"Message:BCC-Display-Name","v4":"message:bcc-display-name"},
+{"v3":"Message:BCC-Email","v4":"message:bcc-email"},
+{"v3":"Message:BCC-Name","v4":"message:bcc-name"},
+{"v3":"Message:CC-Display-Name","v4":"message:cc-display-name"},
+{"v3":"Message:CC-Email","v4":"message:cc-email"},
+{"v3":"Message:CC-Name","v4":"message:cc-name"},
+{"v3":"Message:From-Email","v4":"message:from-email"},
+{"v3":"Message:From-Name","v4":"message:from-name"},
+{"v3":"Message:To-Display-Name","v4":"message:to-display-name"},
+{"v3":"Message:To-Email","v4":"message:to-email"},
+{"v3":"Message:To-Name","v4":"message:to-name"},
+{"v3":"X-TIKA:EXCEPTION:container_exception","v4":"tk:exception:container-exception"},
+{"v3":"X-TIKA:EXCEPTION:embedded_bytes_exception","v4":"tk:exception:embedded-bytes-exception"},
+{"v3":"X-TIKA:EXCEPTION:embedded_exception","v4":"tk:exception:embedded-exception"},
+{"v3":"X-TIKA:EXCEPTION:embedded_stream_exception","v4":"tk:exception:embedded-stream-exception"},
+{"v3":"X-TIKA:EXCEPTION:embedded_warning","v4":"tk:exception:embedded-warning"},
+{"v3":"X-TIKA:EXCEPTION:warn","v4":"tk:exception:warn"},
+{"v3":"X-TIKA:EXCEPTION:write_limit_reached","v4":"tk:exception:write-limit-reached"},
+{"v3":"X-TIKA:Parsed-By","v4":"tk:parsed-by"},
+{"v3":"X-TIKA:Parsed-By-Full-Set","v4":"tk:parsed-by-full-set"},
+{"v3":"X-TIKA:WARN:truncated_metadata","v4":"tk:warn:truncated-metadata"},
+{"v3":"X-TIKA:content","v4":"tk:content"},
+{"v3":"X-TIKA:content_handler","v4":"tk:content-handler"},
+{"v3":"X-TIKA:detectedEncoding","v4":"tk:detected-encoding"},
+{"v3":"X-TIKA:detected_language","v4":"tk:detected-language"},
+{"v3":"X-TIKA:detected_language_confidence","v4":"tk:detected-language-confidence"},
+{"v3":"X-TIKA:detected_language_confidence_raw","v4":"tk:detected-language-confidence-raw"},
+{"v3":"X-TIKA:embedded_depth","v4":"tk:embedded-depth"},
+{"v3":"X-TIKA:embedded_id","v4":"tk:embedded-id"},
+{"v3":"X-TIKA:embedded_id_path","v4":"tk:embedded-id-path"},
+{"v3":"X-TIKA:embedded_resource_path","v4":"tk:embedded-resource-path"},
+{"v3":"X-TIKA:encodingDetector","v4":"tk:encoding-detector"},
+{"v3":"X-TIKA:encrypted","v4":"tk:encrypted"},
+{"v3":"X-TIKA:final_embedded_resource_path","v4":"tk:final-embedded-resource-path"},
+{"v3":"X-TIKA:internalPath","v4":"tk:internal-path"},
+{"v3":"X-TIKA:origResourceName","v4":"tk:orig-resource-name"},
+{"v3":"X-TIKA:parse_time_millis","v4":"tk:parse-time-millis"},
+{"v3":"X-TIKA:pipes_result","v4":"tk:pipes-result"},
+{"v3":"X-TIKA:sourcePath","v4":"tk:source-path"},
+{"v3":"X-TIKA:versionCount","v4":"tk:version-count"},
+{"v3":"X-TIKA:versionNumber","v4":"tk:version-number"},
+{"v3":"access_permission:assemble_document","v4":"access-permission:assemble-document"},
+{"v3":"access_permission:can_modify","v4":"access-permission:can-modify"},
+{"v3":"access_permission:can_print","v4":"access-permission:can-print"},
+{"v3":"access_permission:can_print_faithful","v4":"access-permission:can-print-faithful"},
+{"v3":"access_permission:extract_content","v4":"access-permission:extract-content"},
+{"v3":"access_permission:extract_for_accessibility","v4":"access-permission:extract-for-accessibility"},
+{"v3":"access_permission:fill_in_form","v4":"access-permission:fill-in-form"},
+{"v3":"access_permission:modify_annotations","v4":"access-permission:modify-annotations"},
+{"v3":"database:column_count","v4":"database:column-count"},
+{"v3":"database:column_name","v4":"database:column-name"},
+{"v3":"database:row_count","v4":"database:row-count"},
+{"v3":"database:table_name","v4":"database:table-name"},
+{"v3":"embeddedResourceType","v4":"tk:embedded-resource-type"},
+{"v3":"extended-properties:DocSecurityString","v4":"extended-properties:doc-security-string"},
+{"v3":"extended-properties:HiddedSlides","v4":"extended-properties:HiddenSlides"},
+{"v3":"hasSignature","v4":"tk:has-signature"},
+{"v3":"html_meta:scriptSrc","v4":"html:scriptSrc"},
+{"v3":"imagereader:NumImages","v4":"tk:num-images"},
+{"v3":"meta:mapi-from-representing-email","v4":"mapi:from-representing-email"},
+{"v3":"meta:mapi-from-representing-name","v4":"mapi:from-representing-name"},
+{"v3":"meta:mapi-importance","v4":"mapi:importance"},
+{"v3":"meta:mapi-is-flagged","v4":"mapi:is-flagged"},
+{"v3":"meta:mapi-message-class","v4":"mapi:message-class"},
+{"v3":"meta:mapi-msg-client-submit-time","v4":"mapi:msg-client-submit-time"},
+{"v3":"meta:mapi-recipients-string","v4":"mapi:recipients-string"},
+{"v3":"meta:mapi-sent-by-server-type","v4":"mapi:sent-by-server-type"},
+{"v3":"msoffice:ocxName","v4":"msoffice:ocx-name"},
+{"v3":"msoffice:progID","v4":"msoffice:prog-id"},
+{"v3":"pdf:PDFExtensionVersion","v4":"pdf:pdf-extension-version"},
+{"v3":"pdf:PDFVersion","v4":"pdf:pdf-version"},
+{"v3":"pdf:actionTrigger","v4":"pdf:action-trigger"},
+{"v3":"pdf:actionTriggers","v4":"pdf:action-triggers"},
+{"v3":"pdf:actionTypes","v4":"pdf:action-types"},
+{"v3":"pdf:annotationSubtypes","v4":"pdf:annotation-subtypes"},
+{"v3":"pdf:annotationTypes","v4":"pdf:annotation-types"},
+{"v3":"pdf:associatedFileRelationship","v4":"pdf:associated-file-relationship"},
+{"v3":"pdf:charsPerPage","v4":"pdf:chars-per-page"},
+{"v3":"pdf:containsDamagedFont","v4":"pdf:contains-damaged-font"},
+{"v3":"pdf:containsNonEmbeddedFont","v4":"pdf:contains-non-embedded-font"},
+{"v3":"pdf:docinfo:creator_tool","v4":"pdf:docinfo:creator-tool"},
+{"v3":"pdf:embeddedFileAnnotationType","v4":"pdf:embedded-file-annotation-type"},
+{"v3":"pdf:embeddedFileDescription","v4":"pdf:embedded-file-description"},
+{"v3":"pdf:embeddedFileSubtype","v4":"pdf:embedded-file-subtype"},
+{"v3":"pdf:eofOffsets","v4":"pdf:eof-offsets"},
+{"v3":"pdf:has3D","v4":"pdf:has-3d"},
+{"v3":"pdf:hasAcroFormFields","v4":"pdf:has-acro-form-fields"},
+{"v3":"pdf:hasCollection","v4":"pdf:has-collection"},
+{"v3":"pdf:hasMarkedContent","v4":"pdf:has-marked-content"},
+{"v3":"pdf:hasSignatureFields","v4":"pdf:has-signature-fields"},
+{"v3":"pdf:hasXFA","v4":"pdf:has-xfa"},
+{"v3":"pdf:hasXMP","v4":"pdf:has-xmp"},
+{"v3":"pdf:incrementalUpdateCount","v4":"pdf:incremental-update-count"},
+{"v3":"pdf:incrementalUpdateNumber","v4":"pdf:incremental-update-number"},
+{"v3":"pdf:jsName","v4":"pdf:js-name"},
+{"v3":"pdf:num3DAnnotations","v4":"pdf:num-3d-annotations"},
+{"v3":"pdf:ocrPageCount","v4":"pdf:ocr-page-count"},
+{"v3":"pdf:overallPercentageUnmappedUnicodeChars","v4":"pdf:overall-percentage-unmapped-unicode-chars"},
+{"v3":"pdf:totalUnmappedUnicodeChars","v4":"pdf:total-unmapped-unicode-chars"},
+{"v3":"pdf:unmappedUnicodeCharsPerPage","v4":"pdf:unmapped-unicode-chars-per-page"},
+{"v3":"pdf:xmpLocation","v4":"pdf:xmp-location"},
+{"v3":"pdfa:PDFVersion","v4":"pdfa:pdf-version"},
+{"v3":"pst:discriptorNodeId","v4":"pst:discriptor-node-id"},
+{"v3":"pst:folderPath","v4":"DROPPED"},
+{"v3":"pst:isValid","v4":"pst:is-valid"},
+{"v3":"rendering:Rendered-By","v4":"tk:rendering:rendered-by"},
+{"v3":"rendering:rendering-time-ms","v4":"tk:rendering:rendering-time-ms"},
+{"v3":"rtf_meta:contains_encapsulated_html","v4":"rtf:contains-encapsulated-html"},
+{"v3":"rtf_meta:emb_app_version","v4":"rtf:embedded-app-version"},
+{"v3":"rtf_meta:emb_class","v4":"rtf:embedded-class"},
+{"v3":"rtf_meta:emb_item","v4":"rtf:embedded-item"},
+{"v3":"rtf_meta:emb_topic","v4":"rtf:embedded-topic"},
+{"v3":"rtf_meta:thumbnail","v4":"rtf:thumbnail"},
+{"v3":"signature:contact-info","v4":"tk:signature:contact-info"},
+{"v3":"signature:date","v4":"tk:signature:date"},
+{"v3":"signature:filter","v4":"tk:signature:filter"},
+{"v3":"signature:location","v4":"tk:signature:location"},
+{"v3":"signature:name","v4":"tk:signature:name"},
+{"v3":"signature:reason","v4":"tk:signature:reason"},
+{"v3":"tika_pg:page_number","v4":"tk:page:number"},
+{"v3":"tika_pg:page_rotation","v4":"tk:page:rotation"},
+{"v3":"wordperfect:Build","v4":"quattropro:build"},
+{"v3":"wordperfect:Encrypted","v4":"wordperfect:encrypted"},
+{"v3":"wordperfect:FileId","v4":"wordperfect:file-id"},
+{"v3":"wordperfect:FileSize","v4":"wordperfect:file-size"},
+{"v3":"wordperfect:FileType","v4":"wordperfect:file-type"},
+{"v3":"wordperfect:Id","v4":"quattropro:id"},
+{"v3":"wordperfect:LowestVersion","v4":"quattropro:lowest-version"},
+{"v3":"wordperfect:MajorVersion","v4":"wordperfect:major-version"},
+{"v3":"wordperfect:MinorVersion","v4":"wordperfect:minor-version"},
+{"v3":"wordperfect:ProductType","v4":"wordperfect:product-type"},
+{"v3":"wordperfect:Version","v4":"quattropro:version"}
+]