+ * Invariants enforced: + *
+ * The decorator is a thin passthrough on the happy path: it pushes and pops an + * internal stack on {@code startElement}/{@code endElement} and otherwise forwards + * every event to the wrapped handler unchanged. It deliberately does NOT mask + * bad event sequences (mismatched or excess endElement, duplicate attributes, + * etc.) -- those remain visible to {@link StrictXHTMLValidator} so parser bugs + * still surface as test failures. + *
+ * The unhappy path -- a per-part SAX parser throwing mid-element after emitting + * one or more start tags -- is handled via {@link #drainOpenElements()}, which + * emits a matching {@code endElement} (with the original uri/localName/qName) + * for every element still on the stack, in reverse open order. The wrapped + * handler is left in a well-formed state with no dangling elements from the + * failed sub-parse. + *
+ * Typical use wraps the handler that receives events from an inner SAX parser, + * inside the catch arm that swallows the inner parser's exception: + *
{@code
+ * XHTMLBalancingHandler balancer = new XHTMLBalancingHandler(contentHandler);
+ * try {
+ * XMLReaderUtils.parseSAX(stream, new EmbeddedContentHandler(balancer), context);
+ * } catch (SAXException e) {
+ * balancer.drainOpenElements();
+ * // ... log and continue ...
+ * }
+ * }
+ * This handler does not touch {@code startDocument}/{@code endDocument}; the
+ * caller still owns the document lifecycle.
+ */
+public class XHTMLBalancingHandler extends ContentHandlerDecorator {
+
+ private final Deque+ * Intended for the catch arm of a caller that swallowed a + * {@link SAXException} from an inner SAX parser: the inner parser may have + * left one or more elements open mid-stream, and downstream serialization + * needs matching closers before any further events. + *
+ * Does NOT emit {@code endDocument} -- document lifecycle stays with the + * caller. + */ + public void drainOpenElements() throws SAXException { + while (!openElements.isEmpty()) { + QName q = openElements.pop(); + super.endElement(q.uri, q.localName, q.qName); + } + } + + /** + * Number of elements currently open through this handler. Exposed for + * tests and for callers that want to know whether + * {@link #drainOpenElements()} would emit anything. + */ + public int openElementCount() { + return openElements.size(); + } + + private static final class QName { + final String uri; + final String localName; + final String qName; + + QName(String uri, String localName, String qName) { + this.uri = uri == null ? "" : uri; + this.localName = localName == null ? "" : localName; + this.qName = qName == null ? "" : qName; + } + } +} diff --git a/tika-core/src/test/java/org/apache/tika/sax/XHTMLBalancingHandlerTest.java b/tika-core/src/test/java/org/apache/tika/sax/XHTMLBalancingHandlerTest.java new file mode 100644 index 00000000000..3688616c421 --- /dev/null +++ b/tika-core/src/test/java/org/apache/tika/sax/XHTMLBalancingHandlerTest.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.sax; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; +import org.xml.sax.helpers.AttributesImpl; + +public class XHTMLBalancingHandlerTest { + + private static AttributesImpl noAtts() { + return new AttributesImpl(); + } + + @Test + public void happyPathIsPassthrough() throws Exception { + ToXMLContentHandler out = new ToXMLContentHandler(); + XHTMLBalancingHandler balancer = new XHTMLBalancingHandler(out); + + balancer.startDocument(); + balancer.startElement("", "p", "p", noAtts()); + balancer.characters("hello".toCharArray(), 0, 5); + balancer.endElement("", "p", "p"); + balancer.endDocument(); + + assertEquals(0, balancer.openElementCount()); + // No drain needed -- stack should already be empty. + balancer.drainOpenElements(); + // ToXMLContentHandler.toString() returns the serialized form. + String xml = out.toString(); + assertEquals(true, xml.contains("
hello
"), xml); + } + + @Test + public void drainClosesElementsInReverseOpenOrder() throws Exception { + ToXMLContentHandler out = new ToXMLContentHandler(); + XHTMLBalancingHandler balancer = new XHTMLBalancingHandler(out); + + balancer.startDocument(); + balancer.startElement("", "div", "div", noAtts()); + balancer.startElement("", "p", "p", noAtts()); + balancer.startElement("", "span", "span", noAtts()); + balancer.characters("oops".toCharArray(), 0, 4); + + // Simulate exception mid-element: caller drains. + assertEquals(3, balancer.openElementCount()); + balancer.drainOpenElements(); + assertEquals(0, balancer.openElementCount()); + + balancer.endDocument(); + + String xml = out.toString(); + // Expect in that order. + int spanIdx = xml.indexOf(""); + int pIdx = xml.indexOf(""); + int divIdx = xml.indexOf(""); + assertEquals(true, spanIdx >= 0 && pIdx > spanIdx && divIdx > pIdx, + "expected order, got: " + xml); + } + + @Test + public void drainEmitsMatchingUriAndQName() throws Exception { + // Verifies the Copilot review point: endElement must carry the same + // (uri, localName, qName) tuple as the matching startElement. + ToXMLContentHandler out = new ToXMLContentHandler(); + XHTMLBalancingHandler balancer = new XHTMLBalancingHandler(out); + + balancer.startDocument(); + balancer.startPrefixMapping("h", "http://example.com/ns"); + balancer.startElement("http://example.com/ns", "wrap", "h:wrap", noAtts()); + // Emit content so the serializer can't collapse to a self-closing tag, + // forcing the close form to be explicit -- proves drainOpen used the + // matching qName ("h:wrap") rather than just the local name. + balancer.characters("x".toCharArray(), 0, 1); + balancer.drainOpenElements(); + balancer.endPrefixMapping("h"); + balancer.endDocument(); + + String xml = out.toString(); + assertEquals(true, xml.contains(""), + "expected qualified close , got: " + xml); + } + + @Test + public void drainIsIdempotent() throws Exception { + ToXMLContentHandler out = new ToXMLContentHandler(); + XHTMLBalancingHandler balancer = new XHTMLBalancingHandler(out); + + balancer.startDocument(); + balancer.startElement("", "p", "p", noAtts()); + balancer.drainOpenElements(); + balancer.drainOpenElements(); // second call: no-op + balancer.endDocument(); + assertEquals(0, balancer.openElementCount()); + } + + @Test + public void downstreamValidatorStillCatchesMismatchedEndElement() throws Exception { + // Balancer must NOT silently fix bad happy-path sequences -- the + // StrictXHTMLValidator wrapping the real handler must still see (and + // reject) excess endElement events. + ToXMLContentHandler out = new ToXMLContentHandler(); + StrictXHTMLValidator validator = new StrictXHTMLValidator(out); + XHTMLBalancingHandler balancer = new XHTMLBalancingHandler(validator); + + balancer.startDocument(); + balancer.startElement("", "p", "p", noAtts()); + balancer.endElement("", "p", "p"); + // Extra endElement: stack is empty so balancer pops nothing, but the + // event still flows downstream to the validator, which must throw. + assertThrows(org.xml.sax.SAXException.class, + () -> balancer.endElement("", "p", "p")); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractOfficeParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractOfficeParser.java index ea5179552de..5c7b78c2f7a 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractOfficeParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractOfficeParser.java @@ -112,6 +112,15 @@ public boolean isUseSAXPptxExtractor() { return defaultOfficeParserConfig.isUseSAXPptxExtractor(); } + @Field + public void setPreferAlternateContentChoice(boolean preferAlternateContentChoice) { + defaultOfficeParserConfig.setPreferAlternateContentChoice(preferAlternateContentChoice); + } + + public boolean isPreferAlternateContentChoice() { + return defaultOfficeParserConfig.isPreferAlternateContentChoice(); + } + @Field public void setConcatenatePhoneticRuns(boolean concatenatePhoneticRuns) { defaultOfficeParserConfig.setConcatenatePhoneticRuns(concatenatePhoneticRuns); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java index 5972d548cda..feeaae1b512 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParserConfig.java @@ -35,6 +35,8 @@ public class OfficeParserConfig implements Serializable { private boolean useSAXDocxExtractor = false; private boolean useSAXPptxExtractor = false; + private boolean preferAlternateContentChoice = true; + private boolean extractAllAlternativesFromMSG = false; //we'll stop doing this in 4.x private boolean writeSelectHeadersInBody = true; @@ -169,6 +171,34 @@ public void setUseSAXPptxExtractor(boolean useSAXPptxExtractor) { this.useSAXPptxExtractor = useSAXPptxExtractor; } + /** + * In OOXML, {@code mc:AlternateContent} wraps {@code mc:Choice} (newer/richer + * rendering, e.g. DrawingML text boxes) and {@code mc:Fallback} (degraded VML + * for older consumers). When {@code true} (default), the SAX parser processes + * the Choice branch and skips Fallback. When {@code false}, it processes + * Fallback and skips Choice (legacy behavior prior to Tika 4.x). + *+ * For text extraction, Choice typically contains equal or more content than + * Fallback. + *
+ * Only consulted by the SAX-based (streaming) OOXML extractors. + *
+ * Default: {@code true} + * + * @return whether to prefer mc:Choice over mc:Fallback + */ + public boolean isPreferAlternateContentChoice() { + return preferAlternateContentChoice; + } + + /** + * @param preferAlternateContentChoice whether to prefer mc:Choice over mc:Fallback + * @see #isPreferAlternateContentChoice() + */ + public void setPreferAlternateContentChoice(boolean preferAlternateContentChoice) { + this.preferAlternateContentChoice = preferAlternateContentChoice; + } + public boolean isConcatenatePhoneticRuns() { return concatenatePhoneticRuns; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java index a8d65cd8957..d19aaae4a1d 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java @@ -105,10 +105,14 @@ public abstract class AbstractOOXMLExtractor implements OOXMLExtractor { private final ParseContext context; protected OfficeParserConfig config; protected POIXMLTextExtractor extractor; + //derived from the extractor's OPCPackage; used by the SAX-based extractors and + //SAXBasedMetadataExtractor, which read directly from the package rather than via POI. + protected OPCPackage opcPackage; public AbstractOOXMLExtractor(ParseContext context, POIXMLTextExtractor extractor) { this.context = context; this.extractor = extractor; + this.opcPackage = (extractor == null) ? null : extractor.getPackage(); embeddedExtractor = EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context); // This has already been set by OOXMLParser's call to configure() @@ -116,6 +120,14 @@ public AbstractOOXMLExtractor(ParseContext context, POIXMLTextExtractor extracto this.config = context.get(OfficeParserConfig.class); } + /** + * @return the {@link ParseContext} this extractor was constructed with. Used by SAX-based + * subclasses (e.g. to build a {@link SAXBasedMetadataExtractor}). + */ + protected ParseContext getParseContext() { + return context; + } + /** * @see org.apache.tika.parser.microsoft.ooxml.OOXMLExtractor#getDocument() */ diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/EditType.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/EditType.java new file mode 100644 index 00000000000..cbc9d1ae86a --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/EditType.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +public enum EditType { + NONE, INSERT, DELETE, MOVE_TO, MOVE_FROM +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/FieldCodeParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/FieldCodeParser.java new file mode 100644 index 00000000000..d71cbdc7f9c --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/FieldCodeParser.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Parses OOXML field codes (instrText) to extract URLs from HYPERLINK, + * INCLUDEPICTURE, INCLUDETEXT, IMPORT, and LINK fields. + *
+ * This class has no Tika dependencies and could be contributed to POI.
+ */
+public class FieldCodeParser {
+
+ private static final Pattern HYPERLINK_PATTERN =
+ Pattern.compile("HYPERLINK\\s{1,100}\"([^\"]{1,10000})\"",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern INCLUDEPICTURE_PATTERN =
+ Pattern.compile("INCLUDEPICTURE\\s{1,100}\"([^\"]{1,10000})\"",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern INCLUDETEXT_PATTERN =
+ Pattern.compile("INCLUDETEXT\\s{1,100}\"([^\"]{1,10000})\"",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern IMPORT_PATTERN =
+ Pattern.compile("IMPORT\\s{1,100}\"([^\"]{1,10000})\"",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern LINK_PATTERN =
+ Pattern.compile(
+ "LINK\\s{1,100}[\\w.]{1,50}\\s{1,100}\"([^\"]{1,10000})\"",
+ Pattern.CASE_INSENSITIVE);
+
+ private FieldCodeParser() {
+ }
+
+ /**
+ * Parses a HYPERLINK URL from instrText field code content.
+ * Field codes like: {@code HYPERLINK "https://example.com"}
+ *
+ * @param instrText the accumulated instrText content
+ * @return the URL if found, or null
+ */
+ public static String parseHyperlinkFromInstrText(String instrText) {
+ if (instrText == null || instrText.isEmpty()) {
+ return null;
+ }
+ Matcher m = HYPERLINK_PATTERN.matcher(instrText.trim());
+ if (m.find()) {
+ return m.group(1);
+ }
+ return null;
+ }
+
+ /**
+ * Parses URLs from instrText field codes that reference external resources.
+ * This includes INCLUDEPICTURE, INCLUDETEXT, IMPORT, and LINK fields.
+ *
+ * @param instrText the accumulated instrText content
+ * @param fieldType output parameter - will contain the field type if found
+ * @return the URL if found, or null
+ */
+ public static String parseExternalRefFromInstrText(String instrText,
+ StringBuilder fieldType) {
+ if (instrText == null || instrText.isEmpty()) {
+ return null;
+ }
+ String trimmed = instrText.trim();
+
+ Matcher m = INCLUDEPICTURE_PATTERN.matcher(trimmed);
+ if (m.find()) {
+ fieldType.append("INCLUDEPICTURE");
+ return m.group(1);
+ }
+
+ m = INCLUDETEXT_PATTERN.matcher(trimmed);
+ if (m.find()) {
+ fieldType.append("INCLUDETEXT");
+ return m.group(1);
+ }
+
+ m = IMPORT_PATTERN.matcher(trimmed);
+ if (m.find()) {
+ fieldType.append("IMPORT");
+ return m.group(1);
+ }
+
+ m = LINK_PATTERN.matcher(trimmed);
+ if (m.find()) {
+ fieldType.append("LINK");
+ return m.group(1);
+ }
+
+ return null;
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/FormattingTagManager.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/FormattingTagManager.java
new file mode 100644
index 00000000000..3e8ae2f0de8
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/FormattingTagManager.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.ooxml;
+
+import java.util.Objects;
+
+import org.apache.poi.xwpf.usermodel.UnderlinePatterns;
+import org.xml.sax.SAXException;
+
+import org.apache.tika.sax.XHTMLContentHandler;
+
+/**
+ * Single owner of all run-scoped XHTML wrapper tags, ensuring proper nesting.
+ * Nesting order from outermost to innermost:
+ * {@code text}.
+ *
+ * Hyperlinks come from two OOXML sources with different lifecycles: + *
+ * Used for footnotes, endnotes, and comments so that their content can be
+ * inlined at the point of reference rather than dumped at the end.
+ */
+class OOXMLInlineBodyPartMap {
+
+ static final OOXMLInlineBodyPartMap EMPTY = new OOXMLInlineBodyPartMap(
+ Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(),
+ Collections.emptyMap());
+
+ private final Map
+ * Used for:
+ *
+ * IDs "0" and "-1" are skipped (these are separator/continuation elements in
+ * footnotes/endnotes).
+ */
+class OOXMLPartContentCollector extends DefaultHandler {
+
+ private static final String W_NS =
+ "http://schemas.openxmlformats.org/wordprocessingml/2006/main";
+
+ private final Set
+ * This class has no Tika dependencies and could be contributed to POI.
+ */
+class OOXMLPictureTracker {
+
+ private final Map }, {@code
+ *
+ *
...
private int tableCellDepth = 0;
private int pWithinCell = 0;
+ // Stack of structural elements (paragraphs, tables, rows, cells) this
+ // handler has emitted to the xhtml stream and not yet closed. Used by
+ // closeAnyPending() to drain the stack in reverse order so the captured
+ // XHTML stays balanced when a caller's parseSAX call throws part-way.
+ // Tags emitted by FormattingTagManager (////) are not
+ // tracked here -- closeAnyPending closes them via formattingTags.closeAll()
+ // before draining this stack.
+ private final java.util.Deque},
+ * {@code }, {@code }, or formatting tags on the wire that
+ * collide with the outer {@code