add shim so that tika-eval works for 4.0.0 vs 4.0.0-beta-1 (#2975)

Author: tballison

tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java

@@ -25,8 +25,10 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -159,6 +161,9 @@ public List<Metadata> loadExtract(Path extractFile) throws ExtractReaderExceptio
         try {
             if (fileSuffixes.format == FileSuffixes.FORMAT.JSON) {
                 metadataList = JsonMetadataList.fromJson(reader);
+                for (Metadata m : metadataList) {
+                    normalizeLegacyKeys(m);
+                }
                 if (alterMetadataList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) {
                     while (metadataList.size() > 1) {
                         metadataList.remove(metadataList.size() - 1);
@@ -214,6 +219,77 @@ private List<Metadata> generateListFromTextFile(Reader reader, FileSuffixes file
 
     }
 
+    // Pre-4.0 extract key -> 4.0 key, for the Tika-native fields tika-eval reads. Digest keys are
+    // handled by the prefix rule in normalizeLegacyKeys; Content-Type/Content-Length are standard
+    // names (unchanged) so they are not listed. New-side keys come from the live constants so this
+    // can't drift from the 4.0 declarations.
+    private static final Map<String, String> LEGACY_KEY_MAP = Map.ofEntries(
+            Map.entry("X-TIKA:content", TikaCoreProperties.TIKA_CONTENT.getName()),
+            Map.entry("X-TIKA:content_handler", TikaCoreProperties.TIKA_CONTENT_HANDLER.getName()),
+            Map.entry("X-TIKA:embedded_depth", TikaCoreProperties.EMBEDDED_DEPTH.getName()),
+            Map.entry("X-TIKA:embedded_resource_path", TikaCoreProperties.EMBEDDED_RESOURCE_PATH.getName()),
+            Map.entry("X-TIKA:final_embedded_resource_path", TikaCoreProperties.FINAL_EMBEDDED_RESOURCE_PATH.getName()),
+            Map.entry("X-TIKA:parse_time_millis", TikaCoreProperties.PARSE_TIME_MILLIS.getName()),
+            Map.entry("X-TIKA:resourceName", TikaCoreProperties.RESOURCE_NAME_KEY.getName()),
+            Map.entry("X-TIKA:detectedEncoding", TikaCoreProperties.DETECTED_ENCODING.getName()),
+            Map.entry("X-TIKA:encodingDetector", TikaCoreProperties.ENCODING_DETECTOR.getName()),
+            Map.entry("Content-Type-Hint", TikaCoreProperties.CONTENT_TYPE_HINT.getName()),
+            Map.entry("embeddedResourceType", TikaCoreProperties.EMBEDDED_RESOURCE_TYPE.getName()),
+            Map.entry("X-TIKA:EXCEPTION:container_exception", TikaCoreProperties.CONTAINER_EXCEPTION.getName()),
+            Map.entry("X-TIKA:EXCEPTION:embedded_exception", TikaCoreProperties.EMBEDDED_EXCEPTION.getName()));
+
+    private static final String LEGACY_DIGEST_PREFIX = TikaCoreProperties.LEGACY_TIKA_META_PREFIX
+            + "digest" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
+
+    /**
+     * Pre-4.0 extracts (e.g. 4.0.0-beta-1) key Tika-native fields under X-TIKA:/camelCase names.
+     * Normalize the fields tika-eval reads to their 4.0 tk: keys so a cross-version compare reflects
+     * real diffs, not the rename. Harmless on 4.0 extracts: the legacy keys are simply absent.
+     */
+    private static void normalizeLegacyKeys(Metadata m) {
+        m.setTrusted(true);   // sanctioned trusted transformation: may write reserved tk: keys
+        try {
+            for (Map.Entry<String, String> e : LEGACY_KEY_MAP.entrySet()) {
+                remapLegacyKey(m, e.getKey(), e.getValue());
+            }
+            // digest keys: X-TIKA:digest:<alg> -> tk:digest:<alg> (algorithm unchanged; MD5 drives
+            // embedded-doc matching). names() is a snapshot, so remapping while iterating is safe.
+            for (String name : m.names()) {
+                if (name.startsWith(LEGACY_DIGEST_PREFIX)) {
+                    remapLegacyKey(m, name, TikaCoreProperties.TIKA_META_PREFIX
+                            + name.substring(TikaCoreProperties.LEGACY_TIKA_META_PREFIX.length()));
+                }
+            }
+        } finally {
+            m.setTrusted(false);
+        }
+    }
+
+    private static void remapLegacyKey(Metadata m, String legacyKey, String modernKey) {
+        String[] legacyVals = m.getValues(legacyKey);
+        if (legacyVals.length == 0) {
+            return;
+        }
+        String[] modernVals = m.getValues(modernKey);
+        if (modernVals.length > 0) {
+            // Both present: safe only if identical. Fail loud rather than silently clobber a value.
+            if (!Arrays.equals(legacyVals, modernVals)) {
+                throw new IllegalStateException("Extract has both legacy key '" + legacyKey
+                        + "' and modern key '" + modernKey + "' with different values; legacy-key "
+                        + "normalization would clobber. Extract is inconsistent.");
+            }
+        } else {
+            for (int i = 0; i < legacyVals.length; i++) {
+                if (i == 0) {
+                    m.set(modernKey, legacyVals[i]);
+                } else {
+                    m.add(modernKey, legacyVals[i]);
+                }
+            }
+        }
+        m.remove(legacyKey);
+    }
+
     public enum ALTER_METADATA_LIST {
         AS_IS,  //leave the metadata list as is
         FIRST_ONLY, //take only the metadata list for the "container" document

tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java

@@ -18,6 +18,8 @@
 
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -88,6 +90,38 @@ public void testBasic() throws Exception {
                 .get(TikaCoreProperties.TIKA_CONTENT));
     }
 
+    @Test
+    public void testLegacyKeyNormalization() throws Exception {
+        // a pre-4.0 (4.0.0-beta-1 style) extract: X-TIKA:/camelCase Tika-native keys
+        Path f = getResourceAsFile("/test-dirs/legacy/beta1-style.doc.json").toPath();
+        List<Metadata> list = new ExtractReader().loadExtract(f);
+        assertEquals(2, list.size());
+
+        Metadata container = list.get(0);
+        // readable via the 4.0 Property constants that tika-eval uses
+        assertEquals("the quick brown fox", container.get(TikaCoreProperties.TIKA_CONTENT));
+        assertEquals("12", container.get(TikaCoreProperties.PARSE_TIME_MILLIS));
+        assertEquals("boom", container.get(TikaCoreProperties.CONTAINER_EXCEPTION));
+        assertEquals("abc123", container.get("tk:digest:MD5"));
+        // the legacy keys are gone
+        assertNull(container.get("X-TIKA:content"));
+        assertNull(container.get("X-TIKA:digest:MD5"));
+
+        Metadata embedded = list.get(1);
+        assertEquals("inner.txt", embedded.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
+        assertEquals("1", embedded.get(TikaCoreProperties.EMBEDDED_DEPTH));
+        assertEquals("inner.txt", embedded.get(TikaCoreProperties.RESOURCE_NAME_KEY));
+        assertEquals("ATTACHMENT", embedded.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
+        assertNull(embedded.get("X-TIKA:embedded_resource_path"));
+    }
+
+    @Test
+    public void testLegacyNormalizationCollisionFailsLoud() throws Exception {
+        // both the legacy and modern content key present with different values -> must not clobber
+        Path f = getResourceAsFile("/test-dirs/legacy/collision.doc.json").toPath();
+        assertThrows(IllegalStateException.class, () -> new ExtractReader().loadExtract(f));
+    }
+
     @Test
     public void testTextBasic() throws IOException {
         ExtractReader extractReader = new ExtractReader();

tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/beta1-style.doc.json

@@ -0,0 +1,17 @@
+[
+  {
+    "Content-Type" : "text/plain",
+    "X-TIKA:content" : "the quick brown fox",
+    "X-TIKA:parse_time_millis" : "12",
+    "X-TIKA:digest:MD5" : "abc123",
+    "X-TIKA:EXCEPTION:container_exception" : "boom"
+  },
+  {
+    "Content-Type" : "text/plain",
+    "X-TIKA:embedded_resource_path" : "inner.txt",
+    "X-TIKA:content" : "attachment contents",
+    "X-TIKA:embedded_depth" : "1",
+    "X-TIKA:resourceName" : "inner.txt",
+    "embeddedResourceType" : "ATTACHMENT"
+  }
+]

tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/collision.doc.json

@@ -0,0 +1,7 @@
+[
+  {
+    "Content-Type" : "text/plain",
+    "X-TIKA:content" : "legacy value",
+    "tk:content" : "modern value"
+  }
+]