Pr 3018

CHANGES.txt

@@ -111,6 +111,9 @@ Release 4.0.0 - ???
 
   OTHER CHANGES
 
+   * OneNote extraction now follows document order, omits superseded page
+     revisions, and extracts embedded object BLOBs (TIKA-4814).
+
    * MagicDetector now compiles its regular expression once, in the
      constructor, instead of recompiling it on every match (TIKA-4796).
 

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java

@@ -165,7 +165,7 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata
                     MSOneStorePackage pkg =
                             onenoteParser.parse(alternatePackageOneStoreFile.dataElementPackage);
 
-                    pkg.walkTree(options, metadata, xhtml);
+                    pkg.walkTree(options, metadata, xhtml, context);
                 } catch (Exception e) {
                     OneNoteLegacyDumpStrings dumpStrings =
                             new OneNoteLegacyDumpStrings(oneNoteDirectFileResource, xhtml);

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java

@@ -19,8 +19,24 @@
 /**
  * This class is used to represent the file data.
  */
-class FileDataObject {
+public class FileDataObject {
     public ObjectGroupObjectBLOBDataDeclaration objectDataBLOBDeclaration;
     public ObjectGroupObjectDataBLOBReference objectDataBLOBReference;
     public DataElement objectDataBLOBDataElement;
+
+    /**
+     * @return the opaque binary data of this file data object, or null if it could not be
+     * resolved.
+     */
+    public byte[] getData() {
+        if (objectDataBLOBDataElement != null &&
+                objectDataBLOBDataElement.data instanceof ObjectDataBLOBDataElementData) {
+            ObjectDataBLOBDataElementData blobData =
+                    (ObjectDataBLOBDataElementData) objectDataBLOBDataElement.data;
+            if (blobData.objectDataBLOB != null) {
+                return blobData.objectDataBLOB.getData();
+            }
+        }
+        return null;
+    }
 }

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java

@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.BasicObject;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.BinaryItem;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil;
+
+/**
+ * Specifies an object data BLOB stream object - the opaque binary data of an object,
+ * e.g. an embedded image or file. See MS-FSSHTTPB section 2.2.1.12.8.
+ */
+public class ObjectDataBLOB extends StreamObject {
+    /**
+     * A binary item that holds the opaque binary data.
+     */
+    public BinaryItem data;
+
+    /**
+     * Initializes a new instance of the ObjectDataBLOB class.
+     */
+    public ObjectDataBLOB() {
+        super(StreamObjectTypeHeaderStart.ObjectDataBLOB);
+        this.data = new BinaryItem();
+    }
+
+    /**
+     * @return the opaque binary data as a byte array, or null if not present.
+     */
+    public byte[] getData() {
+        if (this.data == null || this.data.content == null) {
+            return null;
+        }
+        return ByteUtil.toByteArray(this.data.content);
+    }
+
+    /**
+     * Used to de-serialize the element.
+     *
+     * @param byteArray     A Byte array
+     * @param currentIndex  Start position
+     * @param lengthOfItems The length of the items
+     */
+    @Override
+    protected void deserializeItemsFromByteArray(byte[] byteArray, AtomicInteger currentIndex,
+                                                 int lengthOfItems)
+            throws TikaException, IOException {
+        AtomicInteger index = new AtomicInteger(currentIndex.get());
+        this.data = BasicObject.parse(byteArray, index, BinaryItem.class);
+
+        if (index.get() - currentIndex.get() != lengthOfItems) {
+            throw new StreamObjectParseErrorException(currentIndex.get(), "ObjectDataBLOB",
+                    "Stream object over-parse error", null);
+        }
+
+        currentIndex.set(index.get());
+    }
+
+    /**
+     * Used to convert the element into a byte List
+     *
+     * @param byteList A Byte list
+     * @return The number of elements actually contained in the list
+     */
+    @Override
+    protected int serializeItemsToByteList(List<Byte> byteList) throws IOException {
+        int startPoint = byteList.size();
+        if (this.data != null) {
+            byteList.addAll(this.data.serializeToByteList());
+        }
+        return byteList.size() - startPoint;
+    }
+}

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOBDataElementData.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.tika.exception.TikaException;
+
+/**
+ * Object data BLOB data element - carries the opaque binary data of an object, e.g. an
+ * embedded image or file. See MS-FSSHTTPB section 2.2.1.12.8.
+ */
+public class ObjectDataBLOBDataElementData extends DataElementData {
+    public ObjectDataBLOB objectDataBLOB;
+
+    /**
+     * Initializes a new instance of the ObjectDataBLOBDataElementData class.
+     */
+    public ObjectDataBLOBDataElementData() {
+        this.objectDataBLOB = new ObjectDataBLOB();
+    }
+
+    /**
+     * Used to return the length of this element.
+     *
+     * @param byteArray  A Byte array
+     * @param startIndex Start position
+     * @return The element length
+     */
+    @Override
+    public int deserializeDataElementDataFromByteArray(byte[] byteArray, int startIndex)
+            throws TikaException, IOException {
+        AtomicInteger index = new AtomicInteger(startIndex);
+        this.objectDataBLOB = StreamObject.getCurrent(byteArray, index, ObjectDataBLOB.class);
+        return index.get() - startIndex;
+    }
+
+    /**
+     * Used to convert the element into a byte List.
+     *
+     * @return The Byte list
+     */
+    @Override
+    public List<Byte> serializeToByteList() throws TikaException, IOException {
+        return this.objectDataBLOB.serializeToByteList();
+    }
+}

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreCell.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellID;
+
+/**
+ * The revision store content of a single cell (object space), together with the root object
+ * declarations of its current revision. The object groups are ordered from the oldest revision
+ * to the newest.
+ */
+public class RevisionStoreCell {
+    public CellID cellID;
+    public List<RevisionStoreObjectGroup> objectGroups = new ArrayList<>();
+    /**
+     * The effective root object declarations of the cell's current revision, i.e. for each
+     * root role the declaration made by the most recent revision in the base revision chain.
+     */
+    public List<RevisionManifestRootDeclare> rootDeclares = new ArrayList<>();
+}

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java

@@ -18,6 +18,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -39,6 +40,14 @@ public RevisionStoreObjectGroup(ExGuid objectGroupId) {
     public static RevisionStoreObjectGroup createInstance(ExGuid objectGroupId,
                                                           ObjectGroupDataElementData dataObject,
                                                           boolean isEncryption) throws IOException {
+        return createInstance(objectGroupId, dataObject, isEncryption, Collections.emptyMap());
+    }
+
+    public static RevisionStoreObjectGroup createInstance(ExGuid objectGroupId,
+                                                          ObjectGroupDataElementData dataObject,
+                                                          boolean isEncryption,
+                                                          Map<ExGuid, DataElement> blobElements)
+            throws IOException {
         RevisionStoreObjectGroup objectGroup = new RevisionStoreObjectGroup(objectGroupId);
         Map<ExGuid, RevisionStoreObject> objectDict = new HashMap<>();
         if (!isEncryption) {
@@ -63,16 +72,20 @@ public static RevisionStoreObjectGroup createInstance(ExGuid objectGroupId,
                 } else if (objectDeclaration.objectPartitionID.getDecodedValue() == 1) {
                     revisionObject.propertySet =
                             new PropertySetObject(objectDeclaration, objectData);
-                    if (revisionObject.jcid.jcid.isFileData != 0) {
-                        revisionObject.referencedObjectID = objectData.objectExGUIDArray;
-                        revisionObject.referencedObjectSpacesID = objectData.cellIDArray;
-                    }
+                    // the object extended GUID array lists the objects referenced by this
+                    // object, in the same order as the CompactIDs in the OID stream of the
+                    // ObjectSpaceObjectPropSet - see MS-ONESTORE section 2.7.8
+                    revisionObject.referencedObjectID = objectData.objectExGUIDArray;
+                    revisionObject.referencedObjectSpacesID = objectData.cellIDArray;
                 }
             }
 
             for (int i = 0; i <
                     dataObject.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList.size();
                     i++) {
+                if (i >= dataObject.objectGroupData.objectGroupObjectDataBLOBReferenceList.size()) {
+                    throw new IOException("Missing BLOB reference for object declaration " + i);
+                }
                 ObjectGroupObjectBLOBDataDeclaration objectGroupObjectBLOBDataDeclaration =
                         dataObject.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList.get(
                                 i);
@@ -87,11 +100,15 @@ public static RevisionStoreObjectGroup createInstance(ExGuid objectGroupId,
                             objectDict.get(objectGroupObjectBLOBDataDeclaration.objectExGUID);
                 }
                 if (objectGroupObjectBLOBDataDeclaration.objectPartitionID.getDecodedValue() == 2) {
+                    revisionObject.objectID = objectGroupObjectBLOBDataDeclaration.objectExGUID;
+                    revisionObject.objectGroupID = objectGroupId;
                     revisionObject.fileDataObject = new FileDataObject();
                     revisionObject.fileDataObject.objectDataBLOBDeclaration =
                             objectGroupObjectBLOBDataDeclaration;
                     revisionObject.fileDataObject.objectDataBLOBReference =
                             objectGroupObjectDataBLOBReference;
+                    revisionObject.fileDataObject.objectDataBLOBDataElement = blobElements.get(
+                            objectGroupObjectDataBLOBReference.blobExtendedGUID);
                 }
             }
             objectGroup.objects.addAll(objectDict.values());

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java

@@ -19,18 +19,26 @@
 
 import static org.apache.tika.parser.microsoft.onenote.OneNoteParser.ONE_NOTE_PREFIX;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.io.IOException;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.lang3.StringUtils;
 import org.junit.jupiter.api.Test;
+import org.xml.sax.ContentHandler;
 
 import org.apache.tika.TikaTest;
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.ToTextContentHandler;
 
 public class OneNoteParserTest extends TikaTest {
 
@@ -223,6 +231,31 @@ public void testOneNoteEmbeddedWordDoc() throws Exception {
                         ml.get("Content-Type"))));
     }
 
+    @Test
+    public void testOneNoteEmbeddedImage() throws Exception {
+        List<byte[]> embedded = new ArrayList<>();
+        ParseContext context = new ParseContext();
+        context.set(EmbeddedDocumentExtractor.class, new EmbeddedDocumentExtractor() {
+            @Override
+            public boolean shouldParseEmbedded(Metadata metadata) {
+                return true;
+            }
+
+            @Override
+            public void parseEmbedded(TikaInputStream stream, ContentHandler handler,
+                                      Metadata metadata, ParseContext context,
+                                      boolean outputHtml) throws IOException {
+                embedded.add(stream.readAllBytes());
+            }
+        });
+        try (TikaInputStream tis = getResourceAsStream("/test-documents/testOneNoteEmbeddedImage.one")) {
+            new OneNoteParser().parse(tis, new ToTextContentHandler(), new Metadata(), context);
+        }
+
+        assertFalse(embedded.isEmpty());
+        assertTrue(embedded.stream().anyMatch(bytes -> bytes.length > 1000));
+    }
+
     /**
      * Test a document pulled from Office 365 which stores the MS-ONESTORE document using the MS-FSSHTTPB
      * protocol.
@@ -232,6 +265,8 @@ public void testOneNoteDocumentFromOffice365_1() throws Exception {
         Metadata metadata = new Metadata();
         String txt = getText("testOneNoteFromOffice365.one", metadata);
 
+        // only the authors of the current content count - authors that only appear in
+        // older page version snapshots are not reported
         assertEquals(1, metadata.getValues(ONE_NOTE_PREFIX + "mostRecentAuthors").length);
 
         assertEquals(Instant.ofEpochSecond(1636621406),
@@ -241,6 +276,8 @@ public void testOneNoteDocumentFromOffice365_1() throws Exception {
         assertEquals(Instant.ofEpochSecond(1636621448),
                 Instant.ofEpochSecond(Long.parseLong(metadata.get(TikaCoreProperties.MODIFIED))));
         assertContains("Section1Page1Content", txt);
+        // content from revisions other than each cell's current revision manifest
+        assertContains("Section1Page2Content", txt);
     }
 
     /**
@@ -260,12 +297,14 @@ public void testOneNoteDocumentFromOffice365_2() throws Exception {
 
         assertEquals(Instant.ofEpochSecond(1591712300),
                 Instant.ofEpochSecond(Long.parseLong(metadata.get(ONE_NOTE_PREFIX + "creationTimestamp"))));
-        assertEquals(Instant.ofEpochMilli(1623252330000L),
+        assertEquals(Instant.ofEpochMilli(1623597638000L),
                 Instant.ofEpochMilli(Long.parseLong(metadata.get(ONE_NOTE_PREFIX + "lastModifiedTimestamp"))));
         assertEquals(Instant.ofEpochSecond(1623597587),
                 Instant.ofEpochSecond(Long.parseLong(metadata.get(TikaCoreProperties.MODIFIED))));
 
         assertContains("Section1Page1Content", txt);
+        // content from revisions other than each cell's current revision manifest
+        assertContains("Section1Page2Content", txt);
     }
 
     private void assertNoJunk(String txt) {