diff --git a/.skills/metadata-schema.md b/.skills/metadata-schema.md
index 627827eef66..da51bf1fdb5 100644
--- a/.skills/metadata-schema.md
+++ b/.skills/metadata-schema.md
@@ -17,6 +17,19 @@ shows up as a diff). Don't switch to build-time-only generation — that loses t
## Regenerate (after adding/changing a Property or PassthroughPrefix)
+```bash
+tika-metadata-schema/regen.sh
+```
+
+This does the full sequence in one shot: `-am install` so newly added Property/PassthroughPrefix
+classes are on the scan classpath, regenerate all three registries via the forked-exec profile, print
+a before/after key-count check (catches an incomplete classpath scan), `git diff --stat` the
+registries, then run the gate tests. Flags: `--skip-install` (only safe if nothing outside
+`tika-metadata-schema` itself changed since the last install) and `--skip-tests` for a faster inner
+loop. Then review the diff and commit the Property change and the regenerated JSON together.
+
+The manual sequence the script replaces, for reference or if you need to run a step in isolation:
+
```bash
# if parser Property classes changed, install them first so the scan sees them:
./mvnw -Pfast -DskipTests -pl tika-metadata-schema -am install -Dmaven.repo.local=$(pwd)/.local_m2_repo
@@ -24,8 +37,6 @@ shows up as a diff). Don't switch to build-time-only generation — that loses t
./mvnw -pl tika-metadata-schema -Pregen-metadata-schema process-classes -Dmaven.repo.local=$(pwd)/.local_m2_repo
```
-Then `git diff` the JSONs, run the gate tests, commit.
-
**Trap — never use `exec:java`.** `SchemaGenerator` scans `java.class.path`, force-loads Property
classes, and swallows load failures. `exec:java` runs in-process on Maven's classpath → finds zero
keys → emits a near-empty registry that exits 0 and *passes* `MetadataNoUnderscoreTest`. The
diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc
index 3c7ae7a011d..a61ad78ee06 100644
--- a/docs/modules/ROOT/nav.adoc
+++ b/docs/modules/ROOT/nav.adoc
@@ -76,6 +76,7 @@
** xref:advanced/integration-testing/run-uat-script.adoc[Tika-Server REST UAT Script]
* xref:developers/index.adoc[Developers]
** xref:developers/serialization.adoc[Serialization and Configuration]
+** xref:developers/metadata-keys.adoc[Adding a Metadata Key]
* xref:faq.adoc[FAQ]
* xref:security.adoc[Security]
* xref:roadmap.adoc[Roadmap]
diff --git a/docs/modules/ROOT/pages/developers/index.adoc b/docs/modules/ROOT/pages/developers/index.adoc
index e72c12747b9..ac1ef574be9 100644
--- a/docs/modules/ROOT/pages/developers/index.adoc
+++ b/docs/modules/ROOT/pages/developers/index.adoc
@@ -22,6 +22,8 @@ with custom parsers, detectors, and other components.
* xref:developers/serialization.adoc[Serialization and Configuration] - JSON configuration,
@TikaComponent annotation, and creating custom components
+* xref:developers/metadata-keys.adoc[Adding a Metadata Key] - the Property/PassthroughPrefix
+ registry, naming conventions, and regenerating the schema
== Coming Soon
diff --git a/docs/modules/ROOT/pages/developers/metadata-keys.adoc b/docs/modules/ROOT/pages/developers/metadata-keys.adoc
new file mode 100644
index 00000000000..7d5deeb4f78
--- /dev/null
+++ b/docs/modules/ROOT/pages/developers/metadata-keys.adoc
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+= Adding a Metadata Key
+
+Every metadata key Tika can emit is a `Property` constant (or, for runtime-minted names like scraped
+HTML `` tags, a `PassthroughPrefix`) — there are no bare `String` keys. That closed/open key
+space is tracked in a generated, build-gated registry, so adding a key involves one extra step beyond
+writing the Java.
+
+== Add the constant
+
+Add the `Property` to its interface as usual:
+
+[source,java]
+----
+Property MY_NEW_KEY = Property.internalText(TIKA_META_PREFIX + "my-new-key");
+----
+
+Naming conventions (frozen for 4.0):
+
+* Tika-coined keys use the `tk:` namespace, kebab-case, no underscores.
+* External-standard names are used verbatim, including the standard's own prefix (`dc:`, `xmp:`,
+ `cp:`, `extended-properties:`).
+* HTTP headers stay bare — no `http:` namespace (`Content-Type`, `Content-Encoding`, `Location`).
+
+== Regenerate the registry
+
+The registry — three JSON files under `tika-metadata-schema/src/main/resources/`, listing every
+declared key, every open-namespace prefix, and a field-provenance table — is generated from the live
+`Property`/`PassthroughPrefix` declarations, never hand-edited. A committed copy is the reviewable
+audit trail (a rename or dropped key shows up as a diff), and CI fails if it's stale.
+
+Run this after adding, renaming, or removing a `Property` or `PassthroughPrefix`:
+
+[source,bash]
+----
+tika-metadata-schema/regen.sh
+----
+
+It installs the modules the change touched, regenerates all three registry files, sanity-checks the
+diff, and runs the gate tests. Commit the Java change and the regenerated JSON together.
+
+Details, flags, and the traps this script exists to avoid (classpath scanning quirks, `exec:java`
+vs. a forked classpath) are documented in `tika-metadata-schema/README.md`.
+
+== After a rename
+
+The compiler won't catch a stale string literal like `metadata.get("Message-From")`. Grep the repo
+for the old key and replace it with the constant:
+
+[source,bash]
+----
+grep -rn '"Message-' --include=*.java . | grep -v /target/
+----
diff --git a/tika-metadata-schema/README.md b/tika-metadata-schema/README.md
index d2fbc96a172..94968e36fa9 100644
--- a/tika-metadata-schema/README.md
+++ b/tika-metadata-schema/README.md
@@ -36,13 +36,13 @@ declare a `Property` field, force-loads them, reads the global `Property` table,
sorted JSON. `MetadataSchemaTest` regenerates in-memory and asserts it matches the committed file, so
the registry can never drift from the declarations.
-Regenerate after adding/changing a `Property` **or** a `PassthroughPrefix` (writes both files):
+Regenerate after adding/changing a `Property` **or** a `PassthroughPrefix` (writes all three files):
```
-java -cp \
- org.apache.tika.metadata.schema.SchemaGenerator \
- src/main/resources/org/apache/tika/metadata/metadata-keys.json \
- src/main/resources/org/apache/tika/metadata/metadata-open-namespaces.json
+tika-metadata-schema/regen.sh
```
+Installs the dependency modules, regenerates the registries via the forked-exec profile, sanity-checks
+the key-count diff, and runs the gate tests — see `.skills/metadata-schema.md` for flags and the
+manual steps this replaces.
## `metadata-open-namespaces.json` — the open sets (generated + gated)
The **prefixes** under which parsers mint file-controlled key names at runtime — names that are not
diff --git a/tika-metadata-schema/regen.sh b/tika-metadata-schema/regen.sh
new file mode 100755
index 00000000000..4d20fa7f45d
--- /dev/null
+++ b/tika-metadata-schema/regen.sh
@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+#
+# Regenerates and validates the metadata key registry (tika-metadata-schema).
+#
+# Run this after adding, renaming, or removing a Property or PassthroughPrefix
+# constant anywhere in tika-core or the standard parser bundle. It replaces the
+# multi-step manual sequence in .skills/metadata-schema.md with one command:
+# install the dependency modules, regenerate the three registry files, sanity
+# check the diff, then run the gate tests.
+#
+# Usage:
+# tika-metadata-schema/regen.sh [--skip-install] [--skip-tests]
+#
+# --skip-install skip the -am install step (only safe if no Property/
+# PassthroughPrefix classes outside tika-metadata-schema
+# changed since the last install)
+# --skip-tests skip the final gate-test run, for a faster inner loop
+#
+# See tika-metadata-schema/README.md and .skills/metadata-schema.md for the
+# design and the traps this script exists to route around.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+cd "$REPO_ROOT"
+
+MVN_REPO_OPT="-Dmaven.repo.local=$REPO_ROOT/.local_m2_repo"
+
+SKIP_INSTALL=0
+SKIP_TESTS=0
+for arg in "$@"; do
+ case "$arg" in
+ --skip-install) SKIP_INSTALL=1 ;;
+ --skip-tests) SKIP_TESTS=1 ;;
+ -h|--help)
+ sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'
+ exit 0
+ ;;
+ *)
+ echo "Unknown argument: $arg" >&2
+ exit 1
+ ;;
+ esac
+done
+
+REGISTRY_DIR="tika-metadata-schema/src/main/resources/org/apache/tika/metadata"
+REGISTRY_FILES=(
+ "$REGISTRY_DIR/metadata-keys.json"
+ "$REGISTRY_DIR/metadata-open-namespaces.json"
+ "$REGISTRY_DIR/metadata-key-fields.json"
+)
+
+if [ "$SKIP_INSTALL" -eq 0 ]; then
+ echo "==> Installing tika-metadata-schema + its dependency modules (tika-core, standard parsers)"
+ echo " so newly added Property/PassthroughPrefix classes are on the scan classpath."
+ echo " (skip with --skip-install if you already did this)"
+ ./mvnw -Pfast -DskipTests -pl tika-metadata-schema -am install "$MVN_REPO_OPT"
+fi
+
+echo "==> Recording committed key counts, to catch an incomplete classpath scan later"
+BEFORE_COUNTS=()
+for f in "${REGISTRY_FILES[@]}"; do
+ if git cat-file -e "HEAD:$f" 2>/dev/null; then
+ BEFORE_COUNTS+=("$(git show "HEAD:$f" | wc -l)")
+ else
+ BEFORE_COUNTS+=("0")
+ fi
+done
+
+echo "==> Regenerating the registry (forked exec — see .skills/metadata-schema.md for why exec:java is unsafe)"
+./mvnw -pl tika-metadata-schema -Pregen-metadata-schema process-classes "$MVN_REPO_OPT"
+
+echo "==> Comparing key counts before/after (a large drop usually means classes failed to load):"
+for i in "${!REGISTRY_FILES[@]}"; do
+ f="${REGISTRY_FILES[$i]}"
+ before="${BEFORE_COUNTS[$i]}"
+ after=$(wc -l < "$f")
+ line=" $f: $before -> $after lines"
+ if [ "$before" -gt 0 ] && [ "$after" -lt $((before * 90 / 100)) ]; then
+ echo "$line *** WARNING: >10% drop, check --skip-install and module installs ***"
+ else
+ echo "$line"
+ fi
+done
+
+echo "==> git diff of the registries (review before committing):"
+git --no-pager diff --stat -- "${REGISTRY_FILES[@]}"
+
+if [ "$SKIP_TESTS" -eq 0 ]; then
+ echo "==> Running gate tests (MetadataSchemaTest, MetadataFieldTableTest, MetadataNoUnderscoreTest, MetadataCoverageTest, ...)"
+ ./mvnw -pl tika-metadata-schema test "$MVN_REPO_OPT"
+fi
+
+echo "==> Done. Review the diff above, then commit the Property/PassthroughPrefix change and the regenerated JSON together."
diff --git a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataFieldTableTest.java b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataFieldTableTest.java
index 38ca3578f35..9f9c4297e4b 100644
--- a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataFieldTableTest.java
+++ b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataFieldTableTest.java
@@ -41,6 +41,6 @@ public void committedFieldTableMatchesDeclarations() throws Exception {
committed = new String(in.readAllBytes(), StandardCharsets.UTF_8);
}
assertEquals(committed, SchemaGenerator.fieldTable(),
- "metadata-key-fields.json is stale. Run SchemaGenerator.main (3rd arg) and commit it.");
+ "metadata-key-fields.json is stale. Run tika-metadata-schema/regen.sh and commit it.");
}
}
diff --git a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataSchemaTest.java b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataSchemaTest.java
index 25f26c5fbdb..5530b82b102 100644
--- a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataSchemaTest.java
+++ b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataSchemaTest.java
@@ -38,7 +38,8 @@ public class MetadataSchemaTest {
@Test
public void committedKeysMatchDeclarations() throws Exception {
assertEquals(committed(KEYS), SchemaGenerator.generate(),
- "metadata-keys.json is stale. Run SchemaGenerator.main and commit the result.");
+ "metadata-keys.json is stale. Run tika-metadata-schema/regen.sh and commit the "
+ + "result.");
}
@Test
@@ -46,8 +47,8 @@ public void committedOpenNamespacesMatchDeclarations() throws Exception {
// generate() first, so the classpath scan force-loads the PassthroughPrefix declarations.
SchemaGenerator.generate();
assertEquals(committed(OPEN), SchemaGenerator.passthroughJson(),
- "metadata-open-namespaces.json is stale. Run SchemaGenerator.main and commit the "
- + "result.");
+ "metadata-open-namespaces.json is stale. Run tika-metadata-schema/regen.sh and "
+ + "commit the result.");
}
private static String committed(String resource) throws Exception {