Merge branch 'main' of https://gitbox.apache.org/repos/asf/tika

Author: THausherr

pom.xml

@@ -45,6 +45,7 @@
     <module>tika-ml</module>
     <module>tika-encoding-detectors</module>
     <module>tika-parsers</module>
+    <module>tika-metadata-schema</module>
     <module>tika-bundles</module>
     <module>tika-xmp</module>
     <module>tika-langdetect</module>

tika-core/src/main/java/org/apache/tika/metadata/DWG.java

@@ -23,6 +23,9 @@ public interface DWG {
 
     String DWG_PREFIX = "dwg" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
 
+    PassthroughPrefix RAW_FIELD =
+            PassthroughPrefix.file(DWG_PREFIX, "DWGRead JSON header/summary field names");
+
     Property APPLICATION_NAME = Property.externalText(DWG_PREFIX + "applicationName");
 
     Property APPLICATION_VERSION = Property.externalText(DWG_PREFIX + "applicationVersion");

tika-core/src/main/java/org/apache/tika/metadata/HTML.java

@@ -19,6 +19,9 @@
 public interface HTML {
     String PREFIX_HTML_META = "html" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
 
+    PassthroughPrefix SCRAPED_META = PassthroughPrefix.file(PREFIX_HTML_META,
+            "scraped <meta>/http-equiv/OpenGraph names not mapped to a Property");
+
 
     /**
      * If a script element contains a src value, this value

tika-core/src/main/java/org/apache/tika/metadata/MAPI.java

@@ -27,6 +27,7 @@ public interface MAPI {
     String PREFIX_MAPI_META = "mapi" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
     String PREFIX_MAPI_ATTACH_META = "mapi:attach" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
     String PREFIX_MAPI_PROPERTY = PREFIX_MAPI_META + "property" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
+    PassthroughPrefix PROPERTY = PassthroughPrefix.file(PREFIX_MAPI_PROPERTY, "MAPI named properties");
 
     /**
      * MAPI message class.  What type of .msg/MAPI file is it?

tika-core/src/main/java/org/apache/tika/metadata/Message.java

@@ -27,6 +27,9 @@ public interface Message {
     String MESSAGE_RAW_HEADER_PREFIX =
             MESSAGE_PREFIX + "Raw-Header" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
 
+    PassthroughPrefix RAW_HEADER = PassthroughPrefix.file(MESSAGE_RAW_HEADER_PREFIX,
+            "RFC822 / Outlook raw email header names");
+
     String MESSAGE_RECIPIENT_ADDRESS = "Message-Recipient-Address";
 
     String MESSAGE_FROM = "Message-From";

tika-core/src/main/java/org/apache/tika/metadata/Office.java

@@ -37,6 +37,9 @@ public interface Office {
      */
     String USER_DEFINED_METADATA_NAME_PREFIX = "custom:";
 
+    PassthroughPrefix USER_DEFINED = PassthroughPrefix.file(USER_DEFINED_METADATA_NAME_PREFIX,
+            "OOXML/OLE2/ODF user-defined document properties");
+
 
     /**
      * Keywords pertaining to a document. Also populates {@link DublinCore#SUBJECT}.

tika-core/src/main/java/org/apache/tika/metadata/PDF.java

@@ -51,6 +51,9 @@ public interface PDF {
     String PDF_DOC_INFO_CUSTOM_PREFIX =
             PDF_DOC_INFO_PREFIX + "custom" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
 
+    PassthroughPrefix DOC_INFO_CUSTOM =
+            PassthroughPrefix.file(PDF_DOC_INFO_CUSTOM_PREFIX, "PDF Info-dict custom keys");
+
     Property DOC_INFO_CREATED = Property.internalDate(PDF_DOC_INFO_PREFIX + "created");
 
     Property DOC_INFO_CREATOR = Property.internalText(PDF_DOC_INFO_PREFIX + "creator");

tika-core/src/main/java/org/apache/tika/metadata/PassthroughPrefix.java

@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * A Tika-owned prefix under which keys are passed through from the source: the prefix is fixed, but
+ * each key name comes verbatim from the document or tool, so keys are unbounded and can't be
+ * {@link Property} constants. Declaring one self-registers it, so the open set is enumerable (as
+ * {@link Property} makes the closed set) and lintable: a String write is legitimate iff its key is a
+ * registered {@link Property} or its prefix is a registered {@code PassthroughPrefix}.
+ *
+ * @since Apache Tika 4.0.0
+ */
+public final class PassthroughPrefix {
+
+    public enum Provenance { FILE, TOOL }
+
+    private static final Map<String, PassthroughPrefix> REGISTRY = new ConcurrentHashMap<>();
+
+    private final String prefix;
+    private final Provenance provenance;
+    private final String description;
+
+    private PassthroughPrefix(String prefix, Provenance provenance, String description) {
+        this.prefix = prefix;
+        this.provenance = provenance;
+        this.description = description;
+        REGISTRY.put(prefix, this);
+    }
+
+    public static PassthroughPrefix file(String prefix, String description) {
+        return new PassthroughPrefix(prefix, Provenance.FILE, description);
+    }
+
+    public static PassthroughPrefix tool(String prefix, String description) {
+        return new PassthroughPrefix(prefix, Provenance.TOOL, description);
+    }
+
+    /** The full key for a source-derived {@code suffix}. */
+    public String key(String suffix) {
+        return prefix + suffix;
+    }
+
+    public String prefix() {
+        return prefix;
+    }
+
+    public Provenance provenance() {
+        return provenance;
+    }
+
+    public String description() {
+        return description;
+    }
+
+    /** Declared prefixes, from loaded classes only. */
+    public static Collection<PassthroughPrefix> registered() {
+        return Collections.unmodifiableCollection(REGISTRY.values());
+    }
+}

tika-metadata-schema/README.md

@@ -0,0 +1,84 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+# tika-metadata-schema
+
+A machine-readable schema of Apache Tika's metadata keys, plus a registry-driven validator
+(`MetadataKeyValidator`) that classifies any key as CLOSED / OPEN / TEMPLATE / UNKNOWN.
+
+**Scope: `tika-core` + the standard parser bundle.** The heavier/optional parser families
+(scientific, sqlite3, nlp, vlm) are *not* scanned — pulling their runtime deps (netcdf, grib,
+opennlp, DL4J, sqlite-jdbc) into a build-time schema module isn't worth it. Their keys are the only
+ones absent (e.g. `sqlite3:`, `vlm:`, `grib:`, `netcdf:`, `ctakes:`, `NER_`). `MetadataCoverageTest`
+enforces this: any module declaring keys that is neither scanned nor on its explicit out-of-scope
+list fails the build, so nothing escapes *silently*.
+
+## `metadata-keys.json` — the closed set (generated + gated)
+Every key Tika declares as a `Property` constant, plus the bounded digest cross-product
+(`X-TIKA:digest:<ALGORITHM>[:<ENCODING>]`, enumerated from `DigestDef`). Each record:
+`{ key, namespace, valueType, cardinality }`.
+
+**Generated, never hand-edited.** `SchemaGenerator` scans the parser classpath for classes that
+declare a `Property` field, force-loads them, reads the global `Property` table, and writes stable
+sorted JSON. `MetadataSchemaTest` regenerates in-memory and asserts it matches the committed file, so
+the registry can never drift from the declarations.
+
+Regenerate after adding/changing a `Property` **or** a `PassthroughPrefix` (writes both files):
+```
+java -cp <tika-metadata-schema + deps classpath> \
+     org.apache.tika.metadata.schema.SchemaGenerator \
+     src/main/resources/org/apache/tika/metadata/metadata-keys.json \
+     src/main/resources/org/apache/tika/metadata/metadata-open-namespaces.json
+```
+
+## `metadata-open-namespaces.json` — the open sets (generated + gated)
+The **prefixes** under which parsers mint file-controlled key names at runtime — names that are not
+`Property` constants, so the individual keys cannot be enumerated (scraped HTML `<meta>` under
+`html:`, OOXML `custom:`, email `Message:Raw-Header:`, Access `MDB_PROP:`, Vorbis comments, FLV
+attributes, unmapped image/XMP tags, …). Each record: `{ prefix, provenance, description }`.
+
+**Generated from the `PassthroughPrefix` declarations, never hand-edited.** Every such prefix is a
+registered `PassthroughPrefix` constant; `SchemaGenerator` reads that registry the same way it reads
+the `Property` table, and `MetadataSchemaTest` gates it identically. Adding a passthrough prefix in a
+parser and forgetting to regenerate fails the build.
+
+Not covered here: **templates** — parameterized key families like XMP `rdf:Alt` language variants
+`<base-key>:<lang>` (`dc:title:fr`), where the *suffix* rather than the prefix is open. These are
+documented by rule, not enumerated.
+
+## `metadata-string-keys.json` — legacy bare-String closed keys (curated + gated)
+A handful of closed keys predate `Property` and are still declared as bare `String` constants
+(`HttpHeaders.CONTENT_TYPE` = `Content-Type`, the `Content-*`/`Location` family, `Message-*` /
+`Multipart-*`, `tika:chunks`). They self-register nowhere, so the `Property` scan can't see them —
+yet Tika emits them constantly. Each record: `{ key, source }`.
+
+**Curated, but gated against the code:** `MetadataStringKeysTest` reflects each `source` constant
+(e.g. `HttpHeaders.CONTENT_TYPE`) and asserts its live value equals `key`, so a rename/retype/value
+change fails the build. The right long-term fix is to make these `Property` constants (then they'd
+move to `metadata-keys.json` automatically); that's a large, `Content-Type`-blast-radius change left
+for a future major release.
+
+## `MetadataKeyValidator` — the registry-driven lint
+Classifies any key by reading the three registries above (no parser classes needed):
+`CLOSED` (in `metadata-keys.json` or `metadata-string-keys.json`), `OPEN` (under a registered
+passthrough prefix), `TEMPLATE` (a `<closed-key>:<lang>` lang-alt instance), or `UNKNOWN` — a typo,
+an unregistered namespace, or a key nobody declared. This is the payoff the registries exist for: a
+data-driven legitimacy check instead of a hand-coded regex.
+
+Together the files describe the key space of the scanned bundle: closed keys are enumerated and
+gated (Property-backed and legacy-String alike); open namespaces are enumerated by prefix and gated;
+templates are described by rule; and `MetadataCoverageTest` guarantees no scanned-bundle module is
+silently missed. Keys from the out-of-scope families above are excluded by design.

tika-metadata-schema/pom.xml

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.tika</groupId>
+    <artifactId>tika-parent</artifactId>
+    <version>${revision}</version>
+    <relativePath>../tika-parent/pom.xml</relativePath>
+  </parent>
+
+  <artifactId>tika-metadata-schema</artifactId>
+  <name>Apache Tika metadata schema</name>
+  <description>Generates and enforces the machine-readable metadata key registry
+    (metadata-keys.json) from the live Property declarations.</description>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.tika</groupId>
+      <artifactId>tika-core</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <!-- brings the standard parser classes onto the classpath so their Property
+         constants are discoverable by the generator -->
+    <dependency>
+      <groupId>org.apache.tika</groupId>
+      <artifactId>tika-parsers-standard-package</artifactId>
+      <version>${project.version}</version>
+      <type>pom</type>
+    </dependency>
+
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <!-- Regenerate the committed registry with:
+       java -cp <module+deps classpath> org.apache.tika.metadata.schema.SchemaGenerator \
+            src/main/resources/org/apache/tika/metadata/metadata-keys.json
+       MetadataSchemaTest regenerates in-memory and asserts no diff, so CI catches drift. -->
+</project>