TIKA-4837 - flag extra contents in ooxml files

CHANGES.txt

@@ -1,5 +1,15 @@
 Release 4.1.0 - unreleased
 
+   * OOXML parsers flag package parts that are unreachable through the OPC
+     relationship graph: msoffice:has-unreferenced-parts (boolean) and
+     msoffice:unreferenced-part-names. Purely structural (no bytes are
+     inspected; content types come from [Content_Types].xml by extension), so
+     expect false positives from tools that leave orphan parts behind. A hiding
+     place a raw-ZIP scanner can still see, not a statement about what Tika
+     parsed. Applies to Word, Excel, PowerPoint and Visio OOXML (including
+     macro-enabled variants); XPS links content by markup rather than
+     relationships and is not checked (TIKA-4837).
+
    * Shared pipes server (useSharedServer: true, not the default): a client whose
      in-flight parse was killed by another client's restart could restart the
      healthy replacement. ensureRunning holds its lock across the whole fork, so

tika-core/src/main/java/org/apache/tika/metadata/Office.java

@@ -234,6 +234,38 @@ public interface Office {
     // Security-relevant: Framesets can load external URLs
     Property HAS_FRAMESETS = Property.internalBoolean("msoffice:doc:has-framesets");
 
+    /**
+     * Security-relevant, best-effort: true when the OOXML package contains one or more
+     * declared parts that are NOT reachable from the package root through the OPC
+     * relationship graph. Office resolves content by following relationships, so such a
+     * part is dead weight to Office -- a place to hide bytes that a raw-ZIP reader
+     * (AV/DLP/CDR) can still see. Whether <em>Tika</em> parsed it depends on the parser:
+     * some discover parts by content type rather than by relationship (the SAX Word parser
+     * still extracts an unreferenced wordprocessingml part), so this flag says nothing
+     * about which bytes reached a parser.
+     * <p>Purely structural: the part's bytes are never inspected, and its content type comes
+     * from {@code [Content_Types].xml} by extension or explicit override, never by sniffing.
+     * So {@code smuggle.xml} holding RTF is flagged exactly as an innocent orphan is, and
+     * expect false positives -- some producers leave stale customXml or media parts behind
+     * after an edit. (A part with NO declared content type is a different case: POI rejects
+     * the whole package at open, so it surfaces as a container exception, never here.)
+     * <p>This is an informational signal, NOT a guarantee, and it is evadable (a payload
+     * referenced by a relationship type Tika ignores is still reachable and unflagged): per
+     * <a href="https://tika.apache.org/security-model.html">Tika's security model</a>,
+     * Tika is not a security boundary and does not attempt to detect parser differentials.
+     * <p>Set only for Word, Excel, PowerPoint and Visio OOXML (including macro-enabled
+     * variants). XPS links content by markup rather than relationships and is not checked.
+     * See {@link #UNREFERENCED_PART_NAMES} for the part names.
+     */
+    Property HAS_UNREFERENCED_PARTS =
+            Property.internalBoolean("msoffice:has-unreferenced-parts");
+
+    /**
+     * The names of the parts flagged by {@link #HAS_UNREFERENCED_PARTS}.
+     */
+    Property UNREFERENCED_PART_NAMES =
+            Property.internalTextBag("msoffice:unreferenced-part-names");
+
     /**
      * 1-based sheet number for a resource (e.g. an embedded image)
      * anchored to exactly one sheet of a workbook.  For resources

tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json

@@ -317,6 +317,7 @@
   {"class":"org.apache.tika.metadata.Office","field":"HAS_POWER_QUERY","key":"msoffice:excel:has-power-query"},
   {"class":"org.apache.tika.metadata.Office","field":"HAS_SUBDOCUMENTS","key":"msoffice:doc:has-subdocuments"},
   {"class":"org.apache.tika.metadata.Office","field":"HAS_TRACK_CHANGES","key":"msoffice:has-track-changes"},
+  {"class":"org.apache.tika.metadata.Office","field":"HAS_UNREFERENCED_PARTS","key":"msoffice:has-unreferenced-parts"},
   {"class":"org.apache.tika.metadata.Office","field":"HAS_VERY_HIDDEN_SHEETS","key":"msoffice:excel:has-very-hidden-sheets"},
   {"class":"org.apache.tika.metadata.Office","field":"HAS_VML_HYPERLINKS","key":"msoffice:has-vml-hyperlinks"},
   {"class":"org.apache.tika.metadata.Office","field":"HAS_WEB_QUERIES","key":"msoffice:excel:has-web-queries"},
@@ -341,6 +342,7 @@
   {"class":"org.apache.tika.metadata.Office","field":"SLIDE_COUNT","key":"meta:slide-count"},
   {"class":"org.apache.tika.metadata.Office","field":"TABLE_COUNT","key":"meta:table-count"},
   {"class":"org.apache.tika.metadata.Office","field":"UNLISTED_SLIDE_NAMES","key":"msoffice:ppt:unlisted-slide-names"},
+  {"class":"org.apache.tika.metadata.Office","field":"UNREFERENCED_PART_NAMES","key":"msoffice:unreferenced-part-names"},
   {"class":"org.apache.tika.metadata.Office","field":"VERY_HIDDEN_SHEET_NAMES","key":"msoffice:excel:very-hidden-sheet-names"},
   {"class":"org.apache.tika.metadata.Office","field":"WORD_COUNT","key":"meta:word-count"},
   {"class":"org.apache.tika.metadata.Office","field":"WORKBOOK_CODENAME","key":"msoffice:excel:workbook-codename"},

tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json

@@ -366,6 +366,7 @@
   {"key":"msoffice:has-hover-hyperlinks","namespace":"msoffice","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"msoffice:has-linked-ole-objects","namespace":"msoffice","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"msoffice:has-track-changes","namespace":"msoffice","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"},
+  {"key":"msoffice:has-unreferenced-parts","namespace":"msoffice","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"msoffice:has-vml-hyperlinks","namespace":"msoffice","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"msoffice:ocx-name","namespace":"msoffice","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"msoffice:ppt:has-animations","namespace":"msoffice","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"},
@@ -374,6 +375,7 @@
   {"key":"msoffice:ppt:num-unlisted-slides","namespace":"msoffice","valueType":"INTEGER","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"msoffice:ppt:unlisted-slide-names","namespace":"msoffice","valueType":"TEXT","cardinality":"BAG","module":"tika-core"},
   {"key":"msoffice:prog-id","namespace":"msoffice","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
+  {"key":"msoffice:unreferenced-part-names","namespace":"msoffice","valueType":"TEXT","cardinality":"BAG","module":"tika-core"},
   {"key":"multipart:boundary","namespace":"multipart","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"multipart:subtype","namespace":"multipart","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"netcdf:file-type-description","namespace":"netcdf","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-parser-scientific-module"},

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLExtractorFactory.java

@@ -17,14 +17,19 @@
 package org.apache.tika.parser.microsoft.ooxml;
 
 import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.HashSet;
 import java.util.Locale;
 import java.util.Set;
 
 import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
 import org.apache.poi.openxml4j.opc.OPCPackage;
 import org.apache.poi.openxml4j.opc.PackageAccess;
 import org.apache.poi.openxml4j.opc.PackagePart;
+import org.apache.poi.openxml4j.opc.PackageRelationship;
 import org.apache.poi.openxml4j.opc.PackageRelationshipCollection;
+import org.apache.poi.openxml4j.opc.TargetMode;
 import org.apache.poi.util.LocaleUtil;
 import org.apache.poi.xslf.usermodel.XSLFRelation;
 import org.apache.poi.xssf.usermodel.XSSFRelation;
@@ -39,6 +44,7 @@
 import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.HttpHeaders;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Office;
 import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.mime.MediaType;
 import org.apache.tika.parser.EmptyParser;
@@ -118,6 +124,10 @@ public static void parse(TikaInputStream tis, ContentHandler baseHandler, Metada
             }
             metadata.set(HttpHeaders.CONTENT_TYPE, type.toString());
 
+            if (OOXMLParser.OPC_RELATIONSHIP_TYPES.contains(type)) {
+                markUnreferencedParts(pkg, metadata);
+            }
+
             // Detect format and create the appropriate extractor
             String coreContentType = getCorePartContentType(pkg);
 
@@ -216,6 +226,64 @@ private static boolean isVisioType(MediaType type) {
         return type != null && VISIO_SUBTYPES.contains(type.getSubtype());
     }
 
+    private static final String RELATIONSHIPS_CONTENT_TYPE =
+            "application/vnd.openxmlformats-package.relationships+xml";
+
+    /**
+     * Best-effort, security-relevant signal: flags declared parts that are not reachable
+     * from the package root via the OPC relationship graph -- content carried in the file
+     * but outside the structure Office loads by following relationships. Reuses the
+     * already-open package (an in-memory walk of relationships POI loads anyway). Never
+     * throws: a failure here must not break the parse.
+     *
+     * @see Office#HAS_UNREFERENCED_PARTS
+     */
+    private static void markUnreferencedParts(OPCPackage pkg, Metadata metadata) {
+        try {
+            Set<String> reachable = new HashSet<>();
+            Deque<PackagePart> queue = new ArrayDeque<>();
+            addRelatedParts(pkg.getRelationships(), rel -> pkg.getPart(rel), reachable, queue);
+            while (!queue.isEmpty()) {
+                PackagePart part = queue.poll();
+                addRelatedParts(part.getRelationships(), part::getRelatedPart, reachable, queue);
+            }
+            for (PackagePart part : pkg.getParts()) {
+                // relationship parts (_rels/*.rels) are never relationship targets; skip them
+                if (RELATIONSHIPS_CONTENT_TYPE.equals(part.getContentType())) {
+                    continue;
+                }
+                if (!reachable.contains(part.getPartName().getName())) {
+                    metadata.set(Office.HAS_UNREFERENCED_PARTS, true);
+                    metadata.add(Office.UNREFERENCED_PART_NAMES, part.getPartName().getName());
+                }
+            }
+        } catch (Exception e) {
+            // best-effort only; never fail the parse over this signal
+        }
+    }
+
+    @FunctionalInterface
+    private interface PartResolver {
+        PackagePart resolve(PackageRelationship rel) throws Exception;
+    }
+
+    private static void addRelatedParts(PackageRelationshipCollection rels, PartResolver resolver,
+                                        Set<String> reachable, Deque<PackagePart> queue) {
+        for (PackageRelationship rel : rels) {
+            if (rel.getTargetMode() != TargetMode.INTERNAL) {
+                continue;
+            }
+            try {
+                PackagePart part = resolver.resolve(rel);
+                if (part != null && reachable.add(part.getPartName().getName())) {
+                    queue.add(part);
+                }
+            } catch (Exception e) {
+                // unresolved/broken relationship target -- ignore for this best-effort signal
+            }
+        }
+    }
+
     private static String getCorePartContentType(OPCPackage pkg) {
         try {
             PackageRelationshipCollection rels =

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLParser.java

@@ -17,7 +17,6 @@
 package org.apache.tika.parser.microsoft.ooxml;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
@@ -45,48 +44,61 @@ public class OOXMLParser extends AbstractOfficeParser {
     protected static final String SIGNATURE_RELATIONSHIP =
             "http://schemas.openxmlformats.org/package/2006/relationships/digital-signature/origin";
     protected static final MediaType XPS = MediaType.application("vnd.ms-xpsdocument");
-    protected static final Set<MediaType> SUPPORTED_TYPES = Collections.unmodifiableSet(
-            new HashSet<>(Arrays.asList(MediaType
-                            .application("vnd.openxmlformats-officedocument" +
-                                    ".presentationml.presentation"),
-                    MediaType.application("vnd.ms-powerpoint.presentation.macroenabled.12"),
-                    MediaType.application(
-                            "vnd.openxmlformats-officedocument.presentationml.template"),
-                    MediaType.application(
-                                    "vnd.openxmlformats-officedocument.presentationml.slideshow"),
-                    MediaType.application("vnd.ms-powerpoint.slideshow.macroenabled.12"),
-                    MediaType.application("vnd.ms-powerpoint.addin.macroenabled.12"),
-                    MediaType.application("vnd.ms-powerpoint.template.macroenabled.12"),
-                    MediaType.application("vnd.ms-powerpoint.slide.macroenabled.12"),
-                    MediaType.application("vnd.openxmlformats-officedocument.presentationml.slide"),
-
-                    MediaType.application("vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
-                    MediaType.application("vnd.ms-excel.sheet.macroenabled.12"),
-                    MediaType.application(
-                                    "vnd.openxmlformats-officedocument.spreadsheetml.template"),
-                    MediaType.application("vnd.ms-excel.template.macroenabled.12"),
-                    MediaType.application("vnd.ms-excel.addin.macroenabled.12"),
-                    MediaType.application("vnd.ms-excel.sheet.binary.macroenabled.12"),
-
-                    MediaType.application(
-                            "vnd.openxmlformats-officedocument.wordprocessingml.document"),
-                    MediaType.application("vnd.ms-word.document.macroenabled.12"),
-                    MediaType.application(
-                                    "vnd.openxmlformats-officedocument.wordprocessingml.template"),
-                    MediaType.application("vnd.ms-word.template.macroenabled.12"),
-
-                    MediaType.application("vnd.ms-visio.drawing"),
-                    MediaType.application("vnd.ms-visio.drawing.macroenabled.12"),
-                    MediaType.application("vnd.ms-visio.stencil"),
-                    MediaType.application("vnd.ms-visio.stencil.macroenabled.12"),
-                    MediaType.application("vnd.ms-visio.template"),
-                    MediaType.application("vnd.ms-visio.template.macroenabled.12"),
-                    MediaType.application("vnd.ms-visio.drawing"),
-                    MediaType.application("vnd.ms-xpsdocument"),
-                    MediaType.parse("model/vnd.dwfx+xps")
-                    //  MediaType.application("x-tika-ooxml")
-
-            )));
+    private static final Set<MediaType> POWERPOINT_TYPES = Set.of(
+            MediaType.application("vnd.openxmlformats-officedocument.presentationml.presentation"),
+            MediaType.application("vnd.ms-powerpoint.presentation.macroenabled.12"),
+            MediaType.application("vnd.openxmlformats-officedocument.presentationml.template"),
+            MediaType.application("vnd.openxmlformats-officedocument.presentationml.slideshow"),
+            MediaType.application("vnd.ms-powerpoint.slideshow.macroenabled.12"),
+            MediaType.application("vnd.ms-powerpoint.addin.macroenabled.12"),
+            MediaType.application("vnd.ms-powerpoint.template.macroenabled.12"),
+            MediaType.application("vnd.ms-powerpoint.slide.macroenabled.12"),
+            MediaType.application("vnd.openxmlformats-officedocument.presentationml.slide"));
+
+    private static final Set<MediaType> EXCEL_TYPES = Set.of(
+            MediaType.application("vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
+            MediaType.application("vnd.ms-excel.sheet.macroenabled.12"),
+            MediaType.application("vnd.openxmlformats-officedocument.spreadsheetml.template"),
+            MediaType.application("vnd.ms-excel.template.macroenabled.12"),
+            MediaType.application("vnd.ms-excel.addin.macroenabled.12"),
+            MediaType.application("vnd.ms-excel.sheet.binary.macroenabled.12"));
+
+    private static final Set<MediaType> WORD_TYPES = Set.of(
+            MediaType.application("vnd.openxmlformats-officedocument.wordprocessingml.document"),
+            MediaType.application("vnd.ms-word.document.macroenabled.12"),
+            MediaType.application("vnd.openxmlformats-officedocument.wordprocessingml.template"),
+            MediaType.application("vnd.ms-word.template.macroenabled.12"));
+
+    private static final Set<MediaType> VISIO_TYPES = Set.of(
+            MediaType.application("vnd.ms-visio.drawing"),
+            MediaType.application("vnd.ms-visio.drawing.macroenabled.12"),
+            MediaType.application("vnd.ms-visio.stencil"),
+            MediaType.application("vnd.ms-visio.stencil.macroenabled.12"),
+            MediaType.application("vnd.ms-visio.template"),
+            MediaType.application("vnd.ms-visio.template.macroenabled.12"));
+
+    private static final Set<MediaType> XPS_TYPES = Set.of(
+            XPS,
+            MediaType.parse("model/vnd.dwfx+xps"));
+
+    /**
+     * Formats whose content is wired together by OPC relationships, so an unreferenced
+     * part is meaningful. XPS links pages/resources by markup instead.
+     */
+    static final Set<MediaType> OPC_RELATIONSHIP_TYPES =
+            union(WORD_TYPES, EXCEL_TYPES, POWERPOINT_TYPES, VISIO_TYPES);
+
+    protected static final Set<MediaType> SUPPORTED_TYPES =
+            union(OPC_RELATIONSHIP_TYPES, XPS_TYPES);
+
+    @SafeVarargs
+    private static Set<MediaType> union(Set<MediaType>... sets) {
+        Set<MediaType> all = new HashSet<>();
+        for (Set<MediaType> s : sets) {
+            all.addAll(s);
+        }
+        return Collections.unmodifiableSet(all);
+    }
     /**
      * We claim to support all OOXML files, but we actually don't support a small
      * number of them.