TIKA-4663 - make markdown the default content handler in tika-app, tika-server, and the async CLI (#2877)

* TIKA-4663 - make markdown the default content handler in tika-app, tika-server, and the async CLI

Author: tballison

CHANGES.txt

@@ -1,3 +1,15 @@
+Release 4.0.0-beta-1 - unreleased
+
+  BREAKING CHANGES
+
+   * The default content handler is now Markdown. tika-app, tika-server
+     (the /tika and /rmeta endpoints), and the async/pipes CLI now emit
+     Markdown content by default instead of XHTML/XML (plain text for the
+     async CLI). Request the previous format explicitly, e.g. tika-app
+     -x/--xml, the server /tika/xml and /rmeta/xml paths (or the
+     X-Tika-Handler header), and the async CLI --handler x (TIKA-4663).
+
+
 Release 4.0.0-alpha-1 - 5/4/2026
 
   BREAKING CHANGES

docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc

@@ -40,6 +40,16 @@ java -jar tika-app-<version>.jar [option...] [file...]
 
 If you have build scripts or container images that drop in just the jar, update them to unpack the zip and run from inside it.
 
+== Default content handler: XHTML/XML -> Markdown
+
+In 3.x the default content handler produced XHTML/XML. In 4.x the default is **Markdown** everywhere:
+
+* `tika-app` outputs Markdown by default (was XHTML). Pass `-x`/`--xml`, `-h`/`--html`, or `-t`/`--text` to choose another format.
+* `tika-server` — the `/tika` and `/rmeta` endpoints return Markdown content by default (was XHTML/XML). Use an explicit handler path (`/tika/xml`, `/rmeta/xml`, ...) or the `X-Tika-Handler` header to choose another format.
+* The async/pipes CLI emits Markdown by default (was plain text). Use `--handler x` (etc.) to choose another format.
+
+If you parse the extracted content programmatically and expect XHTML/XML, request it explicitly as shown above (TIKA-4663).
+
 == Configuration: XML to JSON
 
 Tika 4.x uses JSON configuration files instead of XML. The legacy `tika-config.xml` format

docs/modules/ROOT/pages/pipes/getting-started.adoc

@@ -53,7 +53,7 @@ java -jar tika-app.jar -i /data/input -o /data/output --handler t
 java -jar tika-app.jar -i /data/input -o /data/output -Z
 ----
 
-Handler types: `t` (text), `h` (html), `x` (xml), `m` (markdown), `b` (body), `i` (ignore/metadata only).
+Handler types: `t` (text), `h` (html), `x` (xml), `m` (markdown), `b` (body), `i` (ignore/metadata only). The default is `m` (markdown).
 
 == JSON Configuration
 

docs/modules/ROOT/pages/pipes/parse-modes.adoc

@@ -69,7 +69,7 @@ and applies to all modes that produce content (`RMETA`, `CONCATENATE`, `CONTENT_
 
 Accepted `type` values: `TEXT`, `HTML`, `XML`, `MARKDOWN`, `BODY`, `IGNORE`. The CLI
 `--handler` flag uses single-letter shortcuts (`t`, `h`, `x`, `m`, `b`, `i`) that map onto
-these values.
+these values. If unset, the default is `MARKDOWN`.
 
 [cols="1,1,2"]
 |===

docs/modules/ROOT/pages/using-tika/cli/index.adoc

@@ -118,7 +118,7 @@ options see <<_tika_pipes_processing,Tika Pipes Processing>> below.
 |Option |Description
 
 |`-x` or `--xml`
-|Output XHTML content (default)
+|Output XHTML content
 
 |`-h` or `--html`
 |Output HTML content
@@ -127,7 +127,7 @@ options see <<_tika_pipes_processing,Tika Pipes Processing>> below.
 |Output plain text content (body)
 
 |`--md`
-|Output Markdown content (body)
+|Output Markdown content (body) (default)
 
 |`-T` or `--text-main`
 |Output plain text — main content only, via the boilerpipe handler
@@ -145,7 +145,7 @@ options see <<_tika_pipes_processing,Tika Pipes Processing>> below.
 |Output metadata in XMP
 
 |`-J` or `--jsonRecursive`
-|Output metadata and content from all embedded files. Combine with `-x`/`-h`/`-t`/`-m` to choose the content type (default: `-x`).
+|Output metadata and content from all embedded files. Combine with `-x`/`-h`/`-t`/`-m` to choose the content type (default: `--md`).
 
 |`-r` or `--pretty-print`
 |For JSON, XML, and XHTML output, add newlines and whitespace for readability.
@@ -378,7 +378,7 @@ This processes all files in the input directory and writes JSON metadata
 |Option |Description
 
 |`--handler=<X>`
-|Content handler type: `t`=text, `h`=html, `x`=xml, `m`=markdown, `b`=body, `i`=ignore. Default: `t`.
+|Content handler type: `t`=text, `h`=html, `x`=xml, `m`=markdown, `b`=body, `i`=ignore. Default: `m`.
 
 |`--concatenate`
 |Concatenate content from all embedded documents into a single content field.

docs/modules/ROOT/pages/using-tika/server/index.adoc

@@ -117,7 +117,7 @@ For the root `/tika` PUT endpoint you can also pick the handler with a header:
 curl -T document.pdf -H "X-Tika-Handler: markdown" http://localhost:9998/tika
 ----
 
-Accepted values: `text`, `html`, `xml`, `markdown` (or `md`), `body`, `ignore`.
+Accepted values: `text`, `html`, `xml`, `markdown` (or `md`), `body`, `ignore`. The default is `markdown`.
 
 === Recursive Metadata (`/rmeta`)
 
@@ -126,7 +126,7 @@ array of metadata objects. The handler controls the content field of each entry:
 
 [source,bash]
 ----
-curl -T document.pdf http://localhost:9998/rmeta            # default: text
+curl -T document.pdf http://localhost:9998/rmeta            # default: markdown
 curl -T document.pdf http://localhost:9998/rmeta/text
 curl -T document.pdf http://localhost:9998/rmeta/html
 curl -T document.pdf http://localhost:9998/rmeta/xml

tika-app/src/main/java/org/apache/tika/cli/AsyncHelper.java

@@ -78,6 +78,10 @@ public static String[] translateArgs(String[] args) {
                 // Translate TikaCLI xml output to the TikaAsyncCLI handler type.
                 argList.add("--handler");
                 argList.add("x");
+            } else if (arg.equals("--md")) {
+                // Translate TikaCLI markdown output to the TikaAsyncCLI handler type.
+                argList.add("--handler");
+                argList.add("m");
             } else if (arg.equals("-J") || arg.equals("--jsonRecursive")) {
                 // TikaAsyncCLI always outputs JSON with recursive metadata (RMETA mode)
                 // This is already the default, so we just skip this arg

tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java

@@ -243,7 +243,7 @@ protected ContentHandler getContentHandler(OutputStream output, Metadata metadat
             return getTransformerHandler(output, "xml", encoding, prettyPrint);
         }
     };
-    private OutputType type = XML;
+    private OutputType type = MARKDOWN;
     private final OutputType HTML = new OutputType() {
         @Override
         protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) throws Exception {
@@ -793,6 +793,8 @@ private ContentHandlerFactory getContentHandlerFactory(OutputType type) {
             handlerType = BasicContentHandlerFactory.HANDLER_TYPE.TEXT;
         } else if (type.equals(TEXT_MAIN)) {
             handlerType = BasicContentHandlerFactory.HANDLER_TYPE.BODY;
+        } else if (type.equals(MARKDOWN)) {
+            handlerType = BasicContentHandlerFactory.HANDLER_TYPE.MARKDOWN;
         } else if (type.equals(METADATA)) {
             handlerType = BasicContentHandlerFactory.HANDLER_TYPE.IGNORE;
         }
@@ -823,18 +825,18 @@ private void usage() {
         out.println("        writing the JSON to stdout. Redirect to save, e.g.:");
         out.println("        --convert-config-xml-to-json=tika-config.xml > tika-config.json");
         out.println("");
-        out.println("    -x  or --xml           Output XHTML content (default)");
+        out.println("    -x  or --xml           Output XHTML content");
         out.println("    -h  or --html          Output HTML content");
         out.println("    -t  or --text          Output plain text content (body)");
-        out.println("    --md                   Output Markdown content (body)");
+        out.println("    --md                   Output Markdown content (body) (default)");
         out.println("    -T  or --text-main     Output plain text content (main content only via boilerpipe handler)");
         out.println("    -A  or --text-all      Output all text content");
         out.println("    -m  or --metadata      Output only metadata");
         out.println("    -j  or --json          Output metadata in JSON");
         out.println("    -y  or --xmp           Output metadata in XMP");
         out.println("    -J  or --jsonRecursive Output metadata and content from all");
         out.println("                           embedded files (choose content type");
-        out.println("                           with -x, -h, -t or -m; default is -x)");
+        out.println("                           with -x, -h, -t or -m; default is --md)");
         out.println("    -a  or --async         Run Tika in async mode; must specify details in a" + " tikaConfig file");
         out.println("    -l  or --language      Output only language");
         out.println("    -d  or --detect        Detect document type");
@@ -915,7 +917,7 @@ private void usage() {
         out.println("    -c, --config=<file>        Tika config file (--config=<file> also accepted)");
         out.println("    -p, --pluginsDir           Plugins directory");
         out.println("    --fileList                 File list (one path per line, relative to -i or absolute)");
-        out.println("    --handler                  Handler type: t=text, h=html, x=xml, m=markdown, b=body, i=ignore");
+        out.println("    --handler                  Handler type: t=text, h=html, x=xml, m=markdown, b=body, i=ignore (default: m)");
         out.println("    --concatenate              Concatenate content from all embedded documents");
         out.println("    --content-only             Output only extracted content (no JSON wrapper); implies --concatenate");
         out.println("    --on-exists                Behavior when an output file exists: exception (default), replace, skip");

tika-app/src/test/java/org/apache/tika/cli/AsyncHelperTest.java

@@ -57,6 +57,13 @@ public void testXmlHandler() throws Exception {
         assertArrayEquals(expected, AsyncHelper.translateArgs(args));
     }
 
+    @Test
+    public void testMarkdownHandler() throws Exception {
+        String[] args = new String[]{"--md", "input", "output"};
+        String[] expected = new String[]{"--handler", "m", "input", "output"};
+        assertArrayEquals(expected, AsyncHelper.translateArgs(args));
+    }
+
     @Test
     public void testExtractLongFormTranslatedToZ() throws Exception {
         // TIKA-4736: tika-app's --extract is the long form of -z. It must be

tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java

@@ -259,7 +259,7 @@ public void testDefaultPDFIncrementalUpdateSettings() throws Exception {
 
     @Test
     public void testExtractJavascript() throws Exception {
-        String json = getParamOutContent("-J", resourcePrefix + "testPDFPackage.pdf");
+        String json = getParamOutContent("-J", "-x", resourcePrefix + "testPDFPackage.pdf");
         assertTrue(json.contains("type=\\\"PDActionJavaScript\\\""));
         assertTrue(json.contains("MACRO"));
         assertTrue(json.contains("NAMES_TREE"));
@@ -341,7 +341,7 @@ public void testDetectOutput() throws Exception {
      */
     @Test
     public void testListMetModels() throws Exception {
-        String content = getParamOutContent("--list-met-models", resourcePrefix + "alice.cli.test");
+        String content = getParamOutContent("--list-met-models", "-x", resourcePrefix + "alice.cli.test");
         assertTrue(content.contains("text/plain"));
     }
 
@@ -663,7 +663,7 @@ public void testDefaultConfigException() throws Exception {
 
     @Test
     public void testConfig() throws Exception {
-        String content = getParamOutContent("--config=" + CONFIGS_DIR.toString() + "/tika-config1.json", resourcePrefix + "bad_xml.xml");
+        String content = getParamOutContent("--config=" + CONFIGS_DIR.toString() + "/tika-config1.json", "-x", resourcePrefix + "bad_xml.xml");
         assertTrue(content.contains("apple"));
         assertTrue(content.contains("org.apache.tika.parser.html.JSoupParser"));
     }
@@ -679,8 +679,12 @@ public void testJsonRecursiveMetadataParserMetadataOnly() throws Exception {
 
     @Test
     public void testJsonRecursiveMetadataParserDefault() throws Exception {
+        // TIKA-4663: default handler is markdown, so recursive content is markdown, not XHTML.
         String content = getParamOutContent("-J", "-r", resourcePrefix + "test_recursive_embedded.docx");
-        assertTrue(content.contains("\"X-TIKA:content\" : \"<html xmlns=\\\"http://www.w3.org/1999/xhtml"));
+        assertFalse(content.contains("<html xmlns=\\\"http://www.w3.org/1999/xhtml"),
+                "default recursive content should be markdown, not XHTML");
+        assertTrue(content.contains("# embed1.zip"),
+                "default recursive content should be markdown (heading syntax)");
     }
 
     @Test