diff --git a/CHANGES.txt b/CHANGES.txt index d2f1ae5bcfc..6563f45e4e6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -111,6 +111,9 @@ Release 4.0.0 - ??? OTHER CHANGES + * OneNote extraction now follows document order, omits superseded page + revisions, and extracts embedded object BLOBs (TIKA-4814). + * MagicDetector now compiles its regular expression once, in the constructor, instead of recompiling it on every match (TIKA-4796). diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java index a9cf1f7ae39..ac5c0c66371 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java @@ -165,7 +165,7 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata MSOneStorePackage pkg = onenoteParser.parse(alternatePackageOneStoreFile.dataElementPackage); - pkg.walkTree(options, metadata, xhtml); + pkg.walkTree(options, metadata, xhtml, context); } catch (Exception e) { OneNoteLegacyDumpStrings dumpStrings = new OneNoteLegacyDumpStrings(oneNoteDirectFileResource, xhtml); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java index 6c1355cb151..8b5a7eabbdf 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java @@ -24,27 +24,40 @@ import java.time.Month; import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.xml.sax.SAXException; +import org.xml.sax.helpers.AttributesImpl; import org.apache.tika.exception.TikaException; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.OneNote; import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.microsoft.onenote.OneNotePropertyEnum; import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.ArrayNumber; import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.EightBytesOfData; import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.FourBytesOfData; import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.IProperty; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtArrayOfPropertyValues; import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtFourBytesOfLengthFollowedByData; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.CellManifestDataElementData; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySet; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestRootDeclare; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreCell; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObject; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObjectGroup; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StorageIndexCellMapping; @@ -58,6 +71,7 @@ import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyType; import org.apache.tika.parser.microsoft.onenote.fsshttpb.unsigned.Unsigned; import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.BitConverter; +import org.apache.tika.sax.EmbeddedContentHandler; import org.apache.tika.sax.XHTMLContentHandler; public class MSOneStorePackage { @@ -76,6 +90,7 @@ public class MSOneStorePackage { private static final Pattern HYPERLINK_PATTERN = Pattern.compile("\uFDDFHYPERLINK\\s+\"([^\"]+)\"([^\"]+)$"); private static final String P = "p"; + private static final int MAX_TRAVERSAL_DEPTH = 100; static { LocalDateTime time32Epoch1980 = LocalDateTime.of(1980, Month.JANUARY, 1, 0, 0); @@ -100,17 +115,25 @@ public class MSOneStorePackage { public List cellManifests; public HeaderCell headerCell; public List dataRoot; + public RevisionStoreCell dataRootCell; public List OtherFileNodeList; - private boolean mostRecentAuthorProp = false; - private boolean originalAuthorProp = false; + /** + * The content cells (object spaces, e.g. pages), each with its object groups and the + * root object declarations of its current revision. + */ + public List cells; private Instant lastModifiedTimestamp = Instant.MIN; private long creationTimestamp = Long.MAX_VALUE; private long lastModified = Long.MIN_VALUE; + private ParseContext parseContext; + private EmbeddedDocumentExtractor embeddedDocumentExtractor; + private Metadata parentMetadata; public MSOneStorePackage() { this.revisionManifests = new ArrayList<>(); this.cellManifests = new ArrayList<>(); this.OtherFileNodeList = new ArrayList<>(); + this.cells = new ArrayList<>(); } /** @@ -159,96 +182,52 @@ private boolean propertyIsBinary(OneNotePropertyEnum property) { property == OneNotePropertyEnum.RichEditTextUnicode; } + /** + * The attribution of an Author property, determined by the property through which the + * author object was referenced. + */ + private enum AuthorRole { + NONE, MOST_RECENT, ORIGINAL + } + public void walkTree(OneNoteTreeWalkerOptions options, Metadata metadata, XHTMLContentHandler xhtml) throws SAXException, TikaException, IOException { - for (RevisionStoreObjectGroup revisionStoreObjectGroup : OtherFileNodeList) { - for (RevisionStoreObject revisionStoreObject : revisionStoreObjectGroup.objects) { - PropertySet propertySet = - revisionStoreObject.propertySet.objectSpaceObjectPropSet.body; - for (int i = 0; i < propertySet.rgData.size(); ++i) { - IProperty property = propertySet.rgData.get(i); - PropertyID propertyID = propertySet.rgPrids[i]; - PropertyType propertyType = PropertyType.fromIntVal(propertyID.type); - OneNotePropertyEnum oneNotePropertyEnum = - OneNotePropertyEnum.of(Unsigned.uint(propertyID.value).longValue()); - if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTimeStamp) { - long fullval = getScalar(property); - Instant instant = Instant.ofEpochSecond( - fullval / 10000000 + DATETIME_EPOCH_DIFF_1601); - if (instant.isAfter(lastModifiedTimestamp)) { - lastModifiedTimestamp = instant; - } - metadata.set(OneNote.LAST_MODIFIED_TIMESTAMP, - String.valueOf(lastModifiedTimestamp.toEpochMilli())); - } else if (oneNotePropertyEnum == OneNotePropertyEnum.CreationTimeStamp) { - // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not - // 1970 - long scalar = getScalar(property); - long creationTs = scalar + TIME32_EPOCH_DIFF_1980; - if (creationTs < creationTimestamp) { - creationTimestamp = creationTs; - } - metadata.set(OneNote.CREATION_TIMESTAMP, String.valueOf(creationTimestamp)); - } else if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTime) { - // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not - // 1970 - long scalar = getScalar(property); - long lastMod = scalar + TIME32_EPOCH_DIFF_1980; - if (lastMod > lastModified) { - lastModified = lastMod; - } - metadata.set(TikaCoreProperties.MODIFIED, String.valueOf(lastModified)); - } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author) { - String author = - new String(((PrtFourBytesOfLengthFollowedByData) property).data, - StandardCharsets.UTF_8); - if (mostRecentAuthorProp) { - mostRecentAuthors.add(author); - } else if (originalAuthorProp) { - originalAuthors.add(author); - } else { - authors.add(author); - } - } else if (oneNotePropertyEnum == OneNotePropertyEnum.AuthorMostRecent) { - mostRecentAuthorProp = true; - } else if (oneNotePropertyEnum == OneNotePropertyEnum.AuthorOriginal) { - originalAuthorProp = true; - } else if (propertyType == PropertyType.FourBytesOfLengthFollowedByData) { - boolean isBinary = propertyIsBinary(oneNotePropertyEnum); - PrtFourBytesOfLengthFollowedByData dataProperty = - (PrtFourBytesOfLengthFollowedByData) property; - if ((dataProperty.data.length & 1) == 0 && - oneNotePropertyEnum != OneNotePropertyEnum.TextExtendedAscii && - !isBinary) { - if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { - xhtml.startElement(P); - xhtml.characters( - new String(dataProperty.data, StandardCharsets.UTF_16LE)); - xhtml.endElement(P); - } - } else if (oneNotePropertyEnum == OneNotePropertyEnum.TextExtendedAscii) { - xhtml.startElement(P); - xhtml.characters( - new String(dataProperty.data, StandardCharsets.US_ASCII)); - xhtml.endElement(P); - } else if (!isBinary) { - if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { - xhtml.startElement(P); - xhtml.characters( - new String(dataProperty.data, StandardCharsets.UTF_16LE)); - xhtml.endElement(P); - } - } else { - if (oneNotePropertyEnum == OneNotePropertyEnum.RichEditTextUnicode) { - handleRichEditTextUnicode(dataProperty.data, xhtml); - } else { - //TODO -- these seem to be somewhat broken font files and other - //odds and ends...what are they and how should we process them? - //handleEmbedded(content.size()); - } - } - } + walkTree(options, metadata, xhtml, new ParseContext()); + } + + public void walkTree(OneNoteTreeWalkerOptions options, Metadata metadata, + XHTMLContentHandler xhtml, ParseContext parseContext) + throws SAXException, TikaException, IOException { + this.parseContext = parseContext; + this.parentMetadata = metadata; + this.embeddedDocumentExtractor = + EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(parseContext); + if (!cells.isEmpty()) { + // Walk each page cell (object space) as a tree, starting from the root objects of + // its current revision and following the object references in property order. This + // emits the text in document order. The pages are walked in the order in which the + // section object space references them; cells that hold older versions of a page + // (the same object space in a different revision context) are skipped. + List pageCells = new ArrayList<>(); + List otherCells = new ArrayList<>(); + splitCells(pageCells, otherCells); + for (RevisionStoreCell cell : pageCells) { + xhtml.startElement("div", "class", "page"); + walkCell(cell, options, metadata, xhtml); + xhtml.endElement("div"); + } + for (RevisionStoreCell cell : otherCells) { + walkCell(cell, options, metadata, xhtml); + } + } else { + // no cell information available - walk the object groups in revision order + Map objectsById = indexObjectsById(OtherFileNodeList); + Set visited = new HashSet<>(); + for (RevisionStoreObjectGroup objectGroup : OtherFileNodeList) { + for (RevisionStoreObject object : objectGroup.objects) { + walkObject(object, objectsById, visited, AuthorRole.NONE, options, metadata, + xhtml, 0); } } } @@ -265,6 +244,435 @@ public void walkTree(OneNoteTreeWalkerOptions options, Metadata metadata, } } + /** + * Splits the cells into page cells, ordered as the section object space references them, + * and the remaining cells. A cell that holds an older version of a page - the same object + * space referenced by a page cell, but in a different revision context - is dropped, so + * content is not emitted once per version snapshot. + */ + private void splitCells(List pageCells, + List otherCells) throws TikaException { + List orderedCellIds = collectSectionReferencedCells(); + if (orderedCellIds.isEmpty()) { + // no page ordering information available - process the cells in storage order + pageCells.addAll(cells); + return; + } + Map remainingCells = new LinkedHashMap<>(); + for (RevisionStoreCell cell : cells) { + remainingCells.put(cell.cellID, cell); + } + Set coveredObjectSpaces = new HashSet<>(); + for (CellID cellId : orderedCellIds) { + RevisionStoreCell cell = remainingCells.remove(cellId); + if (cell != null) { + pageCells.add(cell); + coveredObjectSpaces.add(cellId.extendGUID2); + } + } + for (RevisionStoreCell cell : remainingCells.values()) { + if (cell.cellID == null || + !coveredObjectSpaces.contains(cell.cellID.extendGUID2)) { + // not an older version of one of the pages - keep it so no content is lost + otherCells.add(cell); + } + } + } + + /** + * Walks the section object space (the data root cell) and collects the object space (cell) + * references in document order - this is the order of the pages in the section. + */ + private List collectSectionReferencedCells() throws TikaException { + List orderedCellIds = new ArrayList<>(); + if (dataRootCell == null) { + return orderedCellIds; + } + Map objectsById = indexObjectsById(dataRootCell.objectGroups); + Set visited = new HashSet<>(); + for (RevisionManifestRootDeclare rootDeclare : dataRootCell.rootDeclares) { + collectReferencedCells(objectsById.get(rootDeclare.objectExGuid), objectsById, visited, + orderedCellIds, 0); + } + for (RevisionStoreObjectGroup objectGroup : dataRootCell.objectGroups) { + for (RevisionStoreObject object : objectGroup.objects) { + collectReferencedCells(object, objectsById, visited, orderedCellIds, 0); + } + } + return orderedCellIds; + } + + private void collectReferencedCells(RevisionStoreObject object, + Map objectsById, + Set visited, List out, int depth) + throws TikaException { + if (depth > MAX_TRAVERSAL_DEPTH) { + return; + } + if (object == null || object.propertySet == null || + object.propertySet.objectSpaceObjectPropSet == null) { + return; + } + if (object.objectID != null && !visited.add(object.objectID)) { + return; + } + List actions = collectObjectActions(object); + for (PropertyAction action : actions) { + if (action.spaceReference != null) { + out.add(action.spaceReference); + } else if (action.isChildReference && action.childReference != null) { + collectReferencedCells(objectsById.get(action.childReference), objectsById, + visited, out, depth + 1); + } + } + } + + private void walkCell(RevisionStoreCell cell, OneNoteTreeWalkerOptions options, + Metadata metadata, XHTMLContentHandler xhtml) + throws SAXException, TikaException, IOException { + Map objectsById = indexObjectsById(cell.objectGroups); + Set visited = new HashSet<>(); + // Only objects reachable from the root objects of the current revision are part of + // the current content. The object groups may also contain older, superseded versions + // of objects (under a different object ID); those are intentionally not walked. + for (RevisionManifestRootDeclare rootDeclare : cell.rootDeclares) { + walkObject(objectsById.get(rootDeclare.objectExGuid), objectsById, visited, + AuthorRole.NONE, options, metadata, xhtml, 0); + } + if (visited.isEmpty()) { + // no root objects could be resolved - walk everything so no content is lost + for (RevisionStoreObjectGroup objectGroup : cell.objectGroups) { + for (RevisionStoreObject object : objectGroup.objects) { + walkObject(object, objectsById, visited, AuthorRole.NONE, options, metadata, + xhtml, 0); + } + } + } + } + + /** + * Builds a map of object ID to object. The object groups are ordered from the oldest + * revision to the newest, so a newer version of an object wins over an older one. + */ + private Map indexObjectsById( + List objectGroups) { + Map objectsById = new HashMap<>(); + for (RevisionStoreObjectGroup objectGroup : objectGroups) { + for (RevisionStoreObject object : objectGroup.objects) { + if (object.objectID != null) { + objectsById.put(object.objectID, object); + } + } + } + return objectsById; + } + + private void walkObject(RevisionStoreObject object, + Map objectsById, Set visited, + AuthorRole authorRole, OneNoteTreeWalkerOptions options, + Metadata metadata, XHTMLContentHandler xhtml, int depth) + throws SAXException, TikaException, IOException { + if (depth > MAX_TRAVERSAL_DEPTH) { + throw new TikaException("OneNote object graph exceeds maximum depth of " + + MAX_TRAVERSAL_DEPTH); + } + if (object == null) { + return; + } + if (object.objectID != null && !visited.add(object.objectID)) { + return; + } + if (object.fileDataObject != null) { + // the object carries opaque binary data, e.g. an embedded image or file + handleEmbedded(object.fileDataObject.getData(), xhtml); + } + if (object.propertySet == null || + object.propertySet.objectSpaceObjectPropSet == null) { + return; + } + List actions = collectObjectActions(object); + // An image node can reference the same picture twice: PictureContainer holds the + // canonical image data and WebPictureContainer14 holds a rendition derived from it + // (e.g. re-rendered when the picture was resized). Only emit the derived rendition + // when the canonical container is missing, so the picture is not extracted twice. + boolean hasPrimaryPicture = false; + for (PropertyAction action : actions) { + if (action.isChildReference && action.childReference != null && + action.oneNotePropertyEnum == OneNotePropertyEnum.PictureContainer) { + hasPrimaryPicture = true; + break; + } + } + // The title structure of a page (StructureElementChildNodes) appears above the page + // body on screen, but is declared after the body child nodes. Emit it first so the + // text comes out in visual order. + for (PropertyAction action : actions) { + if (action.oneNotePropertyEnum == OneNotePropertyEnum.StructureElementChildNodes) { + processAction(action, objectsById, visited, authorRole, options, metadata, xhtml, + depth); + } + } + for (PropertyAction action : actions) { + if (hasPrimaryPicture && + action.oneNotePropertyEnum == OneNotePropertyEnum.WebPictureContainer14) { + continue; + } + if (action.oneNotePropertyEnum != OneNotePropertyEnum.StructureElementChildNodes) { + processAction(action, objectsById, visited, authorRole, options, metadata, xhtml, + depth); + } + } + } + + /** + * A property of an object, together with the object reference assigned to it if it is + * an object reference property. + */ + private static final class PropertyAction { + private final IProperty property; + private final PropertyType propertyType; + private final OneNotePropertyEnum oneNotePropertyEnum; + private final boolean isChildReference; + private final ExGuid childReference; + private final CellID spaceReference; + + PropertyAction(IProperty property, PropertyType propertyType, + OneNotePropertyEnum oneNotePropertyEnum, boolean isChildReference, + ExGuid childReference, CellID spaceReference) { + this.property = property; + this.propertyType = propertyType; + this.oneNotePropertyEnum = oneNotePropertyEnum; + this.isChildReference = isChildReference; + this.childReference = childReference; + this.spaceReference = spaceReference; + } + } + + /** + * Flattens the properties of an object, in order, into a list of actions. + */ + private List collectObjectActions(RevisionStoreObject object) + throws TikaException { + List referencedObjects = + object.referencedObjectID == null || object.referencedObjectID.content == null ? + Collections.emptyList() : object.referencedObjectID.content; + List referencedSpaces = object.referencedObjectSpacesID == null || + object.referencedObjectSpacesID.content == null ? Collections.emptyList() : + object.referencedObjectSpacesID.content; + List actions = new ArrayList<>(); + collectActions(object.propertySet.objectSpaceObjectPropSet.body, referencedObjects, + new int[]{0}, referencedSpaces, new int[]{0}, actions, 0); + return actions; + } + + /** + * Flattens the properties of a property set, in order, into a list of actions. + * Properties of type ObjectID or ArrayOfObjectIDs consume, in property order, the object + * references of the containing object, and properties of type ObjectSpaceID or + * ArrayOfObjectSpaceIDs consume the object space (cell) references (see MS-ONESTORE + * section 2.7.8), so the references must be assigned here, in property order, no matter + * in which order the actions are processed later. + */ + private void collectActions(PropertySet propertySet, List referencedObjects, + int[] referenceCursor, List referencedSpaces, + int[] spaceCursor, List actions, int depth) + throws TikaException { + if (depth > MAX_TRAVERSAL_DEPTH) { + throw new TikaException("OneNote property graph exceeds maximum depth of " + + MAX_TRAVERSAL_DEPTH); + } + if (propertySet == null || propertySet.rgPrids == null || propertySet.rgData == null) { + return; + } + for (int i = 0; i < propertySet.rgPrids.length && i < propertySet.rgData.size(); ++i) { + IProperty property = propertySet.rgData.get(i); + PropertyID propertyID = propertySet.rgPrids[i]; + PropertyType propertyType = PropertyType.fromIntVal(propertyID.type); + OneNotePropertyEnum oneNotePropertyEnum = + OneNotePropertyEnum.of(Unsigned.uint(propertyID.value).longValue()); + if (propertyType == PropertyType.ObjectID) { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, true, + nextReference(referencedObjects, referenceCursor), null)); + } else if (propertyType == PropertyType.ArrayOfObjectIDs) { + int count = property instanceof ArrayNumber ? + Math.min(Math.max(0, ((ArrayNumber) property).number), + referencedObjects.size() - referenceCursor[0]) : 0; + for (int j = 0; j < count; ++j) { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, + true, nextReference(referencedObjects, referenceCursor), null)); + } + } else if (propertyType == PropertyType.ObjectSpaceID) { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, false, + null, nextSpaceReference(referencedSpaces, spaceCursor))); + } else if (propertyType == PropertyType.ArrayOfObjectSpaceIDs) { + int count = property instanceof ArrayNumber ? + Math.min(Math.max(0, ((ArrayNumber) property).number), + referencedSpaces.size() - spaceCursor[0]) : 0; + for (int j = 0; j < count; ++j) { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, + false, null, nextSpaceReference(referencedSpaces, spaceCursor))); + } + } else if (propertyType == PropertyType.PropertySet) { + if (property instanceof PropertySet) { + collectActions((PropertySet) property, referencedObjects, referenceCursor, + referencedSpaces, spaceCursor, actions, depth + 1); + } + } else if (propertyType == PropertyType.ArrayOfPropertyValues) { + if (property instanceof PrtArrayOfPropertyValues && + ((PrtArrayOfPropertyValues) property).data != null) { + for (PropertySet nested : ((PrtArrayOfPropertyValues) property).data) { + collectActions(nested, referencedObjects, referenceCursor, + referencedSpaces, spaceCursor, actions, depth + 1); + } + } + } else { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, false, + null, null)); + } + } + } + + private void processAction(PropertyAction action, + Map objectsById, Set visited, + AuthorRole authorRole, OneNoteTreeWalkerOptions options, + Metadata metadata, XHTMLContentHandler xhtml, int depth) + throws SAXException, TikaException, IOException { + if (action.spaceReference != null) { + // a reference to another object space (cell) - cells are walked separately + return; + } + if (action.isChildReference) { + AuthorRole childRole = AuthorRole.NONE; + if (action.oneNotePropertyEnum == OneNotePropertyEnum.AuthorMostRecent) { + childRole = AuthorRole.MOST_RECENT; + } else if (action.oneNotePropertyEnum == OneNotePropertyEnum.AuthorOriginal) { + childRole = AuthorRole.ORIGINAL; + } + walkObject(action.childReference == null ? null : + objectsById.get(action.childReference), objectsById, visited, + childRole, options, metadata, xhtml, depth + 1); + } else { + processPrimitiveProperty(action.property, action.propertyType, + action.oneNotePropertyEnum, authorRole, options, metadata, xhtml); + } + } + + private ExGuid nextReference(List referencedObjects, int[] referenceCursor) { + if (referenceCursor[0] < referencedObjects.size()) { + return referencedObjects.get(referenceCursor[0]++); + } + return null; + } + + private CellID nextSpaceReference(List referencedSpaces, int[] spaceCursor) { + if (spaceCursor[0] < referencedSpaces.size()) { + return referencedSpaces.get(spaceCursor[0]++); + } + return null; + } + + private void processPrimitiveProperty(IProperty property, PropertyType propertyType, + OneNotePropertyEnum oneNotePropertyEnum, + AuthorRole authorRole, + OneNoteTreeWalkerOptions options, Metadata metadata, + XHTMLContentHandler xhtml) + throws SAXException, TikaException, IOException { + if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTimeStamp) { + long fullval = getScalar(property); + Instant instant = Instant.ofEpochSecond( + fullval / 10000000 + DATETIME_EPOCH_DIFF_1601); + if (instant.isAfter(lastModifiedTimestamp)) { + lastModifiedTimestamp = instant; + } + metadata.set(OneNote.LAST_MODIFIED_TIMESTAMP, + String.valueOf(lastModifiedTimestamp.toEpochMilli())); + } else if (oneNotePropertyEnum == OneNotePropertyEnum.CreationTimeStamp) { + // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not + // 1970 + long scalar = getScalar(property); + long creationTs = scalar + TIME32_EPOCH_DIFF_1980; + if (creationTs < creationTimestamp) { + creationTimestamp = creationTs; + } + metadata.set(OneNote.CREATION_TIMESTAMP, String.valueOf(creationTimestamp)); + } else if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTime) { + // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not + // 1970 + long scalar = getScalar(property); + long lastMod = scalar + TIME32_EPOCH_DIFF_1980; + if (lastMod > lastModified) { + lastModified = lastMod; + } + metadata.set(TikaCoreProperties.MODIFIED, String.valueOf(lastModified)); + } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author) { + String author = new String(((PrtFourBytesOfLengthFollowedByData) property).data, + StandardCharsets.UTF_8); + if (authorRole == AuthorRole.MOST_RECENT) { + mostRecentAuthors.add(author); + } else if (authorRole == AuthorRole.ORIGINAL) { + originalAuthors.add(author); + // the original authors are the creators of the content + authors.add(author); + } else { + authors.add(author); + } + } else if (propertyType == PropertyType.FourBytesOfLengthFollowedByData) { + boolean isBinary = propertyIsBinary(oneNotePropertyEnum); + PrtFourBytesOfLengthFollowedByData dataProperty = + (PrtFourBytesOfLengthFollowedByData) property; + if ((dataProperty.data.length & 1) == 0 && + oneNotePropertyEnum != OneNotePropertyEnum.TextExtendedAscii && !isBinary) { + if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { + xhtml.startElement(P); + xhtml.characters(new String(dataProperty.data, StandardCharsets.UTF_16LE)); + xhtml.endElement(P); + } + } else if (oneNotePropertyEnum == OneNotePropertyEnum.TextExtendedAscii) { + xhtml.startElement(P); + xhtml.characters(new String(dataProperty.data, StandardCharsets.US_ASCII)); + xhtml.endElement(P); + } else if (!isBinary) { + if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { + xhtml.startElement(P); + xhtml.characters(new String(dataProperty.data, StandardCharsets.UTF_16LE)); + xhtml.endElement(P); + } + } else { + if (oneNotePropertyEnum == OneNotePropertyEnum.RichEditTextUnicode) { + handleRichEditTextUnicode(dataProperty.data, xhtml); + } else { + //TODO -- these seem to be somewhat broken font files and other + //odds and ends...what are they and how should we process them? + //handleEmbedded(content.size()); + } + } + } + } + + + /** + * Hands the binary data of an embedded object (e.g. an image or an attached file) to the + * embedded document extractor. + */ + private void handleEmbedded(byte[] data, XHTMLContentHandler xhtml) + throws SAXException, IOException { + if (data == null || data.length == 0 || embeddedDocumentExtractor == null) { + return; + } + Metadata embeddedMetadata = Metadata.newInstance(this.parseContext); + AttributesImpl attributes = new AttributesImpl(); + attributes.addAttribute("", "class", "class", "CDATA", "embedded"); + xhtml.startElement("div", attributes); + xhtml.endElement("div"); + try (TikaInputStream tis = TikaInputStream.get(data)) { + if (embeddedDocumentExtractor.shouldParseEmbedded(embeddedMetadata)) { + embeddedDocumentExtractor.parseEmbedded(tis, new EmbeddedContentHandler(xhtml), + embeddedMetadata, this.parseContext, false); + } + } catch (IOException e) { + EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata); + } + } private void handleRichEditTextUnicode(byte[] arr, XHTMLContentHandler xhtml) throws SAXException, IOException, TikaException { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java index 25790ead8b2..30ce905c675 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java @@ -17,9 +17,15 @@ package org.apache.tika.parser.microsoft.onenote.fsshttpb; import java.io.IOException; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -31,6 +37,8 @@ import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestDataElementData; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestObjectGroupReferences; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestRootDeclare; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreCell; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObject; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObjectGroup; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StorageIndexCellMapping; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StorageIndexDataElementData; @@ -40,8 +48,14 @@ import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.DataElementType; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid; import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.HeaderCell; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.GuidUtil; public class MSOneStoreParser { + /** + * The root role declaration used for the encryption key of encrypted sections. + */ + private static final ExGuid ENCRYPTION_KEY_ROOT_EXGUID = + new ExGuid(3, UUID.fromString("4A3717F8-1C14-49E7-9526-81D942DE1741")); private final Set storageIndexHashTab = new HashSet<>(); // The DataElements of Storage Index private List storageIndexDataElements; @@ -55,8 +69,11 @@ public class MSOneStoreParser { private List objectGroupDataElements; // The DataElements of Object BLOB private List objectBlOBElements; + // The DataElements of Object BLOB, keyed by their data element extended GUID + private Map objectBlOBElementsById; public MSOneStorePackage parse(DataElementPackage dataElementPackage) throws IOException { + storageIndexHashTab.clear(); MSOneStorePackage msOneStorePackage = new MSOneStorePackage(); storageIndexDataElements = dataElementPackage.dataElements.stream() @@ -77,6 +94,10 @@ public MSOneStorePackage parse(DataElementPackage dataElementPackage) throws IOE objectBlOBElements = dataElementPackage.dataElements.stream() .filter(d -> d.dataElementType == DataElementType.ObjectDataBLOBDataElementData) .collect(Collectors.toList()); + objectBlOBElementsById = new HashMap<>(); + for (DataElement blobElement : objectBlOBElements) { + objectBlOBElementsById.put(blobElement.dataElementExGuid, blobElement); + } msOneStorePackage.storageIndex = (StorageIndexDataElementData) storageIndexDataElements.get(0).data; @@ -93,29 +114,48 @@ public MSOneStorePackage parse(DataElementPackage dataElementPackage) throws IOE if (headerCellStorageIndexCellMapping != null) { msOneStorePackage.headerCellCellManifest = this.findCellManifest(headerCellStorageIndexCellMapping.cellMappingExGuid); - StorageIndexRevisionMapping headerCellRevisionManifestMapping = - msOneStorePackage.findStorageIndexRevisionMapping( - msOneStorePackage.headerCellCellManifest.cellManifestCurrentRevision - .cellManifestCurrentRevisionExGuid); - msOneStorePackage.headerCellRevisionManifest = this.findRevisionManifestDataElement( - headerCellRevisionManifestMapping.revisionMappingExGuid); - msOneStorePackage.headerCell = - this.parseHeaderCell(msOneStorePackage.headerCellRevisionManifest); - - // Parse Data root - CellID dataRootCellID = - msOneStorePackage.storageManifest.storageManifestRootDeclareList.get(1).cellID; - storageIndexHashTab.add(dataRootCellID); - msOneStorePackage.dataRoot = this.parseObjectGroup(dataRootCellID, msOneStorePackage); - // Parse other data - for (StorageIndexCellMapping storageIndexCellMapping : msOneStorePackage.storageIndex - .storageIndexCellMappingList) { - if (!storageIndexHashTab.contains(storageIndexCellMapping.cellID)) { - msOneStorePackage.OtherFileNodeList.addAll( - this.parseObjectGroup(storageIndexCellMapping.cellID, - msOneStorePackage)); - storageIndexHashTab.add(storageIndexCellMapping.cellID); + if (msOneStorePackage.headerCellCellManifest != null && + msOneStorePackage.headerCellCellManifest.cellManifestCurrentRevision != null) { + StorageIndexRevisionMapping headerCellRevisionManifestMapping = + msOneStorePackage.findStorageIndexRevisionMapping( + msOneStorePackage.headerCellCellManifest.cellManifestCurrentRevision + .cellManifestCurrentRevisionExGuid); + if (headerCellRevisionManifestMapping != null) { + msOneStorePackage.headerCellRevisionManifest = + this.findRevisionManifestDataElement( + headerCellRevisionManifestMapping.revisionMappingExGuid); + if (msOneStorePackage.headerCellRevisionManifest != null) { + msOneStorePackage.headerCell = + this.parseHeaderCell(msOneStorePackage.headerCellRevisionManifest); + } + } + } + } + + // Parse Data root independently of the header-cell metadata. A malformed header cell + // should not prevent valid section cells from being parsed. + CellID dataRootCellID = + msOneStorePackage.storageManifest.storageManifestRootDeclareList.get(1).cellID; + storageIndexHashTab.add(dataRootCellID); + RevisionStoreCell dataRootCell = this.parseCell(dataRootCellID, msOneStorePackage); + if (dataRootCell != null) { + msOneStorePackage.dataRoot = dataRootCell.objectGroups; + msOneStorePackage.dataRootCell = dataRootCell; + } + // Parse other data + for (StorageIndexCellMapping storageIndexCellMapping : msOneStorePackage.storageIndex + .storageIndexCellMappingList) { + if (!storageIndexHashTab.contains(storageIndexCellMapping.cellID)) { + RevisionStoreCell cell = + this.parseCell(storageIndexCellMapping.cellID, msOneStorePackage); + // The storage index can retain a mapping for a deleted version context. + // Such an entry has no CellManifestDataElementData (often its mapping GUID + // is all zero) and therefore cannot contain current document content. + if (cell != null) { + msOneStorePackage.OtherFileNodeList.addAll(cell.objectGroups); + msOneStorePackage.cells.add(cell); } + storageIndexHashTab.add(storageIndexCellMapping.cellID); } } return msOneStorePackage; @@ -128,9 +168,9 @@ public MSOneStorePackage parse(DataElementPackage dataElementPackage) throws IOE * @return The CellManifestDataElementData instance. */ private CellManifestDataElementData findCellManifest(ExGuid cellMappingExtendedGUID) { - return (CellManifestDataElementData) this.cellManifestDataElements.stream() + return this.cellManifestDataElements.stream() .filter(d -> d.dataElementExGuid.equals(cellMappingExtendedGUID)).findFirst() - .orElse(new DataElement()).data; + .map(d -> (CellManifestDataElementData) d.data).orElse(null); } /** @@ -141,9 +181,9 @@ private CellManifestDataElementData findCellManifest(ExGuid cellMappingExtendedG */ private RevisionManifestDataElementData findRevisionManifestDataElement( ExGuid revisionMappingExtendedGUID) { - return (RevisionManifestDataElementData) this.revisionManifestDataElements.stream() + return this.revisionManifestDataElements.stream() .filter(d -> d.dataElementExGuid.equals(revisionMappingExtendedGUID)).findFirst() - .orElse(new DataElement()).data; + .map(d -> (RevisionManifestDataElementData) d.data).orElse(null); } private HeaderCell parseHeaderCell(RevisionManifestDataElementData headerCellRevisionManifest) @@ -159,40 +199,130 @@ private HeaderCell parseHeaderCell(RevisionManifestDataElementData headerCellRev return HeaderCell.createInstance((ObjectGroupDataElementData) element.data); } - private List parseObjectGroup(CellID objectGroupCellID, - MSOneStorePackage msOneStorePackage) + private RevisionStoreCell parseCell(CellID objectGroupCellID, + MSOneStorePackage msOneStorePackage) throws IOException { StorageIndexCellMapping storageIndexCellMapping = msOneStorePackage.findStorageIndexCellMapping(objectGroupCellID); + if (storageIndexCellMapping == null) { + return null; + } CellManifestDataElementData cellManifest = this.findCellManifest(storageIndexCellMapping.cellMappingExGuid); + if (cellManifest == null || cellManifest.cellManifestCurrentRevision == null) { + return null; + } List objectGroups = new ArrayList<>(); msOneStorePackage.cellManifests.add(cellManifest); StorageIndexRevisionMapping revisionMapping = msOneStorePackage.findStorageIndexRevisionMapping( cellManifest.cellManifestCurrentRevision.cellManifestCurrentRevisionExGuid); + if (revisionMapping == null) { + return null; + } RevisionManifestDataElementData revisionManifest = findRevisionManifestDataElement(revisionMapping.revisionMappingExGuid); - msOneStorePackage.revisionManifests.add(revisionManifest); - RevisionManifestRootDeclare encryptionKeyRoot = - revisionManifest.revisionManifestRootDeclareList.stream() - .filter(r -> r.rootExGuid.equals(new ExGuid(3, - UUID.fromString("4A3717F8-1C14-49E7-9526-81D942DE1741")))) - .findFirst().orElse(null); - boolean isEncryption = encryptionKeyRoot != null; - for (RevisionManifestObjectGroupReferences objRef : - revisionManifest.revisionManifestObjectGroupReferences) { - ObjectGroupDataElementData dataObject = - (ObjectGroupDataElementData) objectGroupDataElements.stream() - .filter(d -> d.dataElementExGuid.equals(objRef.objectGroupExtendedGUID)) - .findFirst().get().data; - - RevisionStoreObjectGroup objectGroup = - RevisionStoreObjectGroup.createInstance(objRef.objectGroupExtendedGUID, - dataObject, isEncryption); - objectGroups.add(objectGroup); + if (revisionManifest == null || revisionManifest.revisionManifest == null) { + return null; + } + + // A revision manifest may only reference the object groups that were created or + // modified in that revision. The remaining object groups belong to the chain of + // base revisions (MS-FSSHTTPB "Base Revision ID"). Follow that chain and collect + // the object groups of every revision, oldest revision first, so no content is lost. + Deque revisionChain = new ArrayDeque<>(); + Set seenRevisionIds = new HashSet<>(); + while (revisionManifest != null) { + ExGuid revisionId = revisionManifest.revisionManifest.revisionID; + if (revisionId != null && !seenRevisionIds.add(revisionId)) { + // cycle guard - stop if we have already visited this revision + break; + } + revisionChain.addFirst(revisionManifest); + ExGuid baseRevisionId = revisionManifest.revisionManifest.baseRevisionID; + if (baseRevisionId == null || baseRevisionId.guid == null || + GuidUtil.emptyGuid().equals(baseRevisionId.guid)) { + break; + } + StorageIndexRevisionMapping baseRevisionMapping = + msOneStorePackage.findStorageIndexRevisionMapping(baseRevisionId); + revisionManifest = baseRevisionMapping == null ? null : + findRevisionManifestDataElement(baseRevisionMapping.revisionMappingExGuid); + } + + Set seenObjectGroupIds = new HashSet<>(); + // for each root role, the declaration made by the most recent revision wins + Map effectiveRootDeclares = new LinkedHashMap<>(); + for (RevisionManifestDataElementData manifest : revisionChain) { + msOneStorePackage.revisionManifests.add(manifest); + RevisionManifestRootDeclare encryptionKeyRoot = + manifest.revisionManifestRootDeclareList.stream() + .filter(r -> r.rootExGuid.equals(ENCRYPTION_KEY_ROOT_EXGUID)) + .findFirst().orElse(null); + boolean isEncryption = encryptionKeyRoot != null; + for (RevisionManifestRootDeclare rootDeclare : + manifest.revisionManifestRootDeclareList) { + if (!rootDeclare.rootExGuid.equals(ENCRYPTION_KEY_ROOT_EXGUID)) { + effectiveRootDeclares.put(rootDeclare.rootExGuid, rootDeclare); + } + } + for (RevisionManifestObjectGroupReferences objRef : + manifest.revisionManifestObjectGroupReferences) { + if (!seenObjectGroupIds.add(objRef.objectGroupExtendedGUID)) { + continue; + } + Optional dataElement = objectGroupDataElements.stream() + .filter(d -> d.dataElementExGuid.equals(objRef.objectGroupExtendedGUID)) + .findFirst(); + if (!dataElement.isPresent()) { + continue; + } + ObjectGroupDataElementData dataObject = + (ObjectGroupDataElementData) dataElement.get().data; + + RevisionStoreObjectGroup objectGroup = + RevisionStoreObjectGroup.createInstance(objRef.objectGroupExtendedGUID, + dataObject, isEncryption, objectBlOBElementsById); + objectGroups.add(objectGroup); + } } - return objectGroups; + removeSupersededObjects(objectGroups); + + RevisionStoreCell cell = new RevisionStoreCell(); + cell.cellID = objectGroupCellID; + cell.objectGroups = objectGroups; + cell.rootDeclares = new ArrayList<>(effectiveRootDeclares.values()); + return cell; + } + + /** + * An object that is modified in a later revision appears again, with the same object ID, + * in that revision's object group. Keep only the newest version of each object, replacing + * the older version in place so the original object ordering is preserved. + * + * @param objectGroups The object groups ordered from the oldest revision to the newest. + */ + private void removeSupersededObjects(List objectGroups) { + Map> containingList = new HashMap<>(); + Map indexInList = new HashMap<>(); + for (RevisionStoreObjectGroup objectGroup : objectGroups) { + List objects = objectGroup.objects; + for (int i = 0; i < objects.size(); ) { + RevisionStoreObject object = objects.get(i); + ExGuid objectId = object.objectID; + if (objectId != null && containingList.containsKey(objectId)) { + // newer version of an already seen object - replace the older one in place + containingList.get(objectId).set(indexInList.get(objectId), object); + objects.remove(i); + } else { + if (objectId != null) { + containingList.put(objectId, objects); + indexInList.put(objectId, i); + } + ++i; + } + } + } } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java index 1211d2bf9d1..d3a70780a9a 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java @@ -19,8 +19,24 @@ /** * This class is used to represent the file data. */ -class FileDataObject { +public class FileDataObject { public ObjectGroupObjectBLOBDataDeclaration objectDataBLOBDeclaration; public ObjectGroupObjectDataBLOBReference objectDataBLOBReference; public DataElement objectDataBLOBDataElement; + + /** + * @return the opaque binary data of this file data object, or null if it could not be + * resolved. + */ + public byte[] getData() { + if (objectDataBLOBDataElement != null && + objectDataBLOBDataElement.data instanceof ObjectDataBLOBDataElementData) { + ObjectDataBLOBDataElementData blobData = + (ObjectDataBLOBDataElementData) objectDataBLOBDataElement.data; + if (blobData.objectDataBLOB != null) { + return blobData.objectDataBLOB.getData(); + } + } + return null; + } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java new file mode 100644 index 00000000000..e92d04f6e9f --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.tika.exception.TikaException; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.BasicObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.BinaryItem; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil; + +/** + * Specifies an object data BLOB stream object - the opaque binary data of an object, + * e.g. an embedded image or file. See MS-FSSHTTPB section 2.2.1.12.8. + */ +public class ObjectDataBLOB extends StreamObject { + /** + * A binary item that holds the opaque binary data. + */ + public BinaryItem data; + + /** + * Initializes a new instance of the ObjectDataBLOB class. + */ + public ObjectDataBLOB() { + super(StreamObjectTypeHeaderStart.ObjectDataBLOB); + this.data = new BinaryItem(); + } + + /** + * @return the opaque binary data as a byte array, or null if not present. + */ + public byte[] getData() { + if (this.data == null || this.data.content == null) { + return null; + } + return ByteUtil.toByteArray(this.data.content); + } + + /** + * Used to de-serialize the element. + * + * @param byteArray A Byte array + * @param currentIndex Start position + * @param lengthOfItems The length of the items + */ + @Override + protected void deserializeItemsFromByteArray(byte[] byteArray, AtomicInteger currentIndex, + int lengthOfItems) + throws TikaException, IOException { + AtomicInteger index = new AtomicInteger(currentIndex.get()); + this.data = BasicObject.parse(byteArray, index, BinaryItem.class); + + if (index.get() - currentIndex.get() != lengthOfItems) { + throw new StreamObjectParseErrorException(currentIndex.get(), "ObjectDataBLOB", + "Stream object over-parse error", null); + } + + currentIndex.set(index.get()); + } + + /** + * Used to convert the element into a byte List + * + * @param byteList A Byte list + * @return The number of elements actually contained in the list + */ + @Override + protected int serializeItemsToByteList(List byteList) throws IOException { + int startPoint = byteList.size(); + if (this.data != null) { + byteList.addAll(this.data.serializeToByteList()); + } + return byteList.size() - startPoint; + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOBDataElementData.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOBDataElementData.java new file mode 100644 index 00000000000..ec16e4ca31d --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOBDataElementData.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.tika.exception.TikaException; + +/** + * Object data BLOB data element - carries the opaque binary data of an object, e.g. an + * embedded image or file. See MS-FSSHTTPB section 2.2.1.12.8. + */ +public class ObjectDataBLOBDataElementData extends DataElementData { + public ObjectDataBLOB objectDataBLOB; + + /** + * Initializes a new instance of the ObjectDataBLOBDataElementData class. + */ + public ObjectDataBLOBDataElementData() { + this.objectDataBLOB = new ObjectDataBLOB(); + } + + /** + * Used to return the length of this element. + * + * @param byteArray A Byte array + * @param startIndex Start position + * @return The element length + */ + @Override + public int deserializeDataElementDataFromByteArray(byte[] byteArray, int startIndex) + throws TikaException, IOException { + AtomicInteger index = new AtomicInteger(startIndex); + this.objectDataBLOB = StreamObject.getCurrent(byteArray, index, ObjectDataBLOB.class); + return index.get() - startIndex; + } + + /** + * Used to convert the element into a byte List. + * + * @return The Byte list + */ + @Override + public List serializeToByteList() throws TikaException, IOException { + return this.objectDataBLOB.serializeToByteList(); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreCell.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreCell.java new file mode 100644 index 00000000000..188a2a45de9 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreCell.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellID; + +/** + * The revision store content of a single cell (object space), together with the root object + * declarations of its current revision. The object groups are ordered from the oldest revision + * to the newest. + */ +public class RevisionStoreCell { + public CellID cellID; + public List objectGroups = new ArrayList<>(); + /** + * The effective root object declarations of the cell's current revision, i.e. for each + * root role the declaration made by the most recent revision in the base revision chain. + */ + public List rootDeclares = new ArrayList<>(); +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java index afd2f6c067c..dfa3f05d8f3 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -39,6 +40,14 @@ public RevisionStoreObjectGroup(ExGuid objectGroupId) { public static RevisionStoreObjectGroup createInstance(ExGuid objectGroupId, ObjectGroupDataElementData dataObject, boolean isEncryption) throws IOException { + return createInstance(objectGroupId, dataObject, isEncryption, Collections.emptyMap()); + } + + public static RevisionStoreObjectGroup createInstance(ExGuid objectGroupId, + ObjectGroupDataElementData dataObject, + boolean isEncryption, + Map blobElements) + throws IOException { RevisionStoreObjectGroup objectGroup = new RevisionStoreObjectGroup(objectGroupId); Map objectDict = new HashMap<>(); if (!isEncryption) { @@ -63,16 +72,20 @@ public static RevisionStoreObjectGroup createInstance(ExGuid objectGroupId, } else if (objectDeclaration.objectPartitionID.getDecodedValue() == 1) { revisionObject.propertySet = new PropertySetObject(objectDeclaration, objectData); - if (revisionObject.jcid.jcid.isFileData != 0) { - revisionObject.referencedObjectID = objectData.objectExGUIDArray; - revisionObject.referencedObjectSpacesID = objectData.cellIDArray; - } + // the object extended GUID array lists the objects referenced by this + // object, in the same order as the CompactIDs in the OID stream of the + // ObjectSpaceObjectPropSet - see MS-ONESTORE section 2.7.8 + revisionObject.referencedObjectID = objectData.objectExGUIDArray; + revisionObject.referencedObjectSpacesID = objectData.cellIDArray; } } for (int i = 0; i < dataObject.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList.size(); i++) { + if (i >= dataObject.objectGroupData.objectGroupObjectDataBLOBReferenceList.size()) { + throw new IOException("Missing BLOB reference for object declaration " + i); + } ObjectGroupObjectBLOBDataDeclaration objectGroupObjectBLOBDataDeclaration = dataObject.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList.get( i); @@ -87,11 +100,15 @@ public static RevisionStoreObjectGroup createInstance(ExGuid objectGroupId, objectDict.get(objectGroupObjectBLOBDataDeclaration.objectExGUID); } if (objectGroupObjectBLOBDataDeclaration.objectPartitionID.getDecodedValue() == 2) { + revisionObject.objectID = objectGroupObjectBLOBDataDeclaration.objectExGUID; + revisionObject.objectGroupID = objectGroupId; revisionObject.fileDataObject = new FileDataObject(); revisionObject.fileDataObject.objectDataBLOBDeclaration = objectGroupObjectBLOBDataDeclaration; revisionObject.fileDataObject.objectDataBLOBReference = objectGroupObjectDataBLOBReference; + revisionObject.fileDataObject.objectDataBLOBDataElement = blobElements.get( + objectGroupObjectDataBLOBReference.blobExtendedGUID); } } objectGroup.objects.addAll(objectDict.values()); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java index bab0a6fbcb0..f4876ad989d 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java @@ -19,18 +19,26 @@ import static org.apache.tika.parser.microsoft.onenote.OneNoteParser.ONE_NOTE_PREFIX; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.Test; +import org.xml.sax.ContentHandler; import org.apache.tika.TikaTest; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.ToTextContentHandler; public class OneNoteParserTest extends TikaTest { @@ -223,6 +231,31 @@ public void testOneNoteEmbeddedWordDoc() throws Exception { ml.get("Content-Type")))); } + @Test + public void testOneNoteEmbeddedImage() throws Exception { + List embedded = new ArrayList<>(); + ParseContext context = new ParseContext(); + context.set(EmbeddedDocumentExtractor.class, new EmbeddedDocumentExtractor() { + @Override + public boolean shouldParseEmbedded(Metadata metadata) { + return true; + } + + @Override + public void parseEmbedded(TikaInputStream stream, ContentHandler handler, + Metadata metadata, ParseContext context, + boolean outputHtml) throws IOException { + embedded.add(stream.readAllBytes()); + } + }); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testOneNoteEmbeddedImage.one")) { + new OneNoteParser().parse(tis, new ToTextContentHandler(), new Metadata(), context); + } + + assertFalse(embedded.isEmpty()); + assertTrue(embedded.stream().anyMatch(bytes -> bytes.length > 1000)); + } + /** * Test a document pulled from Office 365 which stores the MS-ONESTORE document using the MS-FSSHTTPB * protocol. @@ -232,6 +265,8 @@ public void testOneNoteDocumentFromOffice365_1() throws Exception { Metadata metadata = new Metadata(); String txt = getText("testOneNoteFromOffice365.one", metadata); + // only the authors of the current content count - authors that only appear in + // older page version snapshots are not reported assertEquals(1, metadata.getValues(ONE_NOTE_PREFIX + "mostRecentAuthors").length); assertEquals(Instant.ofEpochSecond(1636621406), @@ -241,6 +276,8 @@ public void testOneNoteDocumentFromOffice365_1() throws Exception { assertEquals(Instant.ofEpochSecond(1636621448), Instant.ofEpochSecond(Long.parseLong(metadata.get(TikaCoreProperties.MODIFIED)))); assertContains("Section1Page1Content", txt); + // content from revisions other than each cell's current revision manifest + assertContains("Section1Page2Content", txt); } /** @@ -260,12 +297,14 @@ public void testOneNoteDocumentFromOffice365_2() throws Exception { assertEquals(Instant.ofEpochSecond(1591712300), Instant.ofEpochSecond(Long.parseLong(metadata.get(ONE_NOTE_PREFIX + "creationTimestamp")))); - assertEquals(Instant.ofEpochMilli(1623252330000L), + assertEquals(Instant.ofEpochMilli(1623597638000L), Instant.ofEpochMilli(Long.parseLong(metadata.get(ONE_NOTE_PREFIX + "lastModifiedTimestamp")))); assertEquals(Instant.ofEpochSecond(1623597587), Instant.ofEpochSecond(Long.parseLong(metadata.get(TikaCoreProperties.MODIFIED)))); assertContains("Section1Page1Content", txt); + // content from revisions other than each cell's current revision manifest + assertContains("Section1Page2Content", txt); } private void assertNoJunk(String txt) { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java new file mode 100644 index 00000000000..f5a44440533 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.StringWriter; +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; +import org.xml.sax.ContentHandler; + +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.DataElement; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.FileDataObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectDataBLOB; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectDataBLOBDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectBLOBDataDeclaration; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectDataBLOBReference; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectDeclare; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObjectGroup; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StreamObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StreamObjectParseErrorException; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.BinaryItem; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.DataElementType; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil; +import org.apache.tika.sax.ToTextContentHandler; +import org.apache.tika.sax.XHTMLContentHandler; + +/** + * Tests for object data BLOBs (embedded images and files) in the MS-FSSHTTPB packaged + * revision store. + */ +public class MSOneStoreBlobTest { + + private static final byte[] BLOB_BYTES = + "these are the bytes of an embedded image".getBytes(StandardCharsets.UTF_8); + + /** + * An object data BLOB data element must survive a serialization round trip. Before + * ObjectDataBLOBDataElementData existed, deserializing a data element of this type threw, + * so any OneNote file with an embedded image or file lost its structure entirely. + */ + @Test + public void testObjectDataBLOBDataElementRoundTrip() throws Exception { + DataElement reparsed = roundTripBlobElement(); + assertEquals(DataElementType.ObjectDataBLOBDataElementData, reparsed.dataElementType); + assertArrayEquals(BLOB_BYTES, + ((ObjectDataBLOBDataElementData) reparsed.data).objectDataBLOB.getData()); + } + + /** + * A revision store object whose BLOB reference resolves to an object data BLOB element + * must expose the BLOB bytes and hand them to the embedded document extractor during the + * tree walk. + */ + @Test + public void testBlobIsEmittedAsEmbeddedDocument() throws Exception { + DataElement blobElement = roundTripBlobElement(); + + ObjectGroupObjectBLOBDataDeclaration declaration = + new ObjectGroupObjectBLOBDataDeclaration(); + declaration.objectExGUID = new ExGuid(1, UUID.randomUUID()); + declaration.objectPartitionID.setDecodedValue(2); + ObjectGroupObjectDataBLOBReference blobReference = + new ObjectGroupObjectDataBLOBReference(); + blobReference.blobExtendedGUID = blobElement.dataElementExGuid; + + ObjectGroupDataElementData groupData = new ObjectGroupDataElementData(); + groupData.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList + .add(declaration); + groupData.objectGroupData.objectGroupObjectDataBLOBReferenceList.add(blobReference); + + RevisionStoreObjectGroup objectGroup = + RevisionStoreObjectGroup.createInstance(new ExGuid(2, UUID.randomUUID()), + groupData, false, + Collections.singletonMap(blobElement.dataElementExGuid, blobElement)); + assertEquals(1, objectGroup.objects.size()); + assertArrayEquals(BLOB_BYTES, objectGroup.objects.get(0).fileDataObject.getData()); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.OtherFileNodeList.add(objectGroup); + + List embedded = new ArrayList<>(); + ParseContext context = new ParseContext(); + context.set(EmbeddedDocumentExtractor.class, new EmbeddedDocumentExtractor() { + @Override + public boolean shouldParseEmbedded(Metadata metadata) { + return true; + } + + @Override + public void parseEmbedded(TikaInputStream stream, ContentHandler handler, + Metadata metadata, ParseContext context, + boolean outputHtml) throws IOException { + embedded.add(stream.readAllBytes()); + } + }); + + Metadata metadata = new Metadata(); + XHTMLContentHandler xhtml = + new XHTMLContentHandler(new ToTextContentHandler(new StringWriter()), metadata, + context); + xhtml.startDocument(); + pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, xhtml, context); + xhtml.endDocument(); + + assertEquals(1, embedded.size()); + assertArrayEquals(BLOB_BYTES, embedded.get(0)); + } + + @Test + public void testMissingBlobDataAndMalformedLengthAreHandled() throws Exception { + FileDataObject fileData = new FileDataObject(); + assertNull(fileData.getData()); + + DataElement wrongType = new DataElement(); + fileData.objectDataBLOBDataElement = wrongType; + assertNull(fileData.getData()); + + ObjectDataBLOBDataElementData emptyBlobData = new ObjectDataBLOBDataElementData(); + emptyBlobData.objectDataBLOB = null; + wrongType.data = emptyBlobData; + assertNull(fileData.getData()); + + ObjectDataBLOB blob = new ObjectDataBLOB(); + blob.data = null; + assertNull(blob.getData()); + blob.data = new BinaryItem(); + blob.data.content = null; + assertNull(blob.getData()); + blob.data = new BinaryItem(); + blob.data.content.add((byte) 1); + // Exercise serialization of a non-null BLOB; the round-trip assertion is above. + blob.serializeToByteList(); + + ObjectDataBLOB validBlob = new ObjectDataBLOB(); + validBlob.data.content.add((byte) 1); + byte[] itemBytes = ByteUtil.toByteArray(validBlob.data.serializeToByteList()); + AtomicInteger index = new AtomicInteger(0); + java.lang.reflect.Method deserialize = ObjectDataBLOB.class.getDeclaredMethod( + "deserializeItemsFromByteArray", byte[].class, AtomicInteger.class, int.class); + deserialize.setAccessible(true); + InvocationTargetException exception = assertThrows(InvocationTargetException.class, + () -> deserialize.invoke(new ObjectDataBLOB(), itemBytes, index, + itemBytes.length + 1)); + assertTrue(exception.getCause() instanceof StreamObjectParseErrorException); + } + + @Test + public void testEmptyAndFailedEmbeddedDataAreIgnored() throws Exception { + MSOneStorePackage emptyPackage = packageWithFileData(new ObjectDataBLOBDataElementData()); + walkWithExtractor(emptyPackage, new EmbeddedDocumentExtractor() { + @Override + public boolean shouldParseEmbedded(Metadata metadata) { + throw new AssertionError("empty data must not reach the extractor"); + } + + @Override + public void parseEmbedded(TikaInputStream stream, ContentHandler handler, + Metadata metadata, ParseContext context, + boolean outputHtml) { + } + }); + + DataElement blobElement = roundTripBlobElement(); + MSOneStorePackage failedPackage = packageWithFileData( + (ObjectDataBLOBDataElementData) blobElement.data); + walkWithExtractor(failedPackage, new EmbeddedDocumentExtractor() { + @Override + public boolean shouldParseEmbedded(Metadata metadata) { + return true; + } + + @Override + public void parseEmbedded(TikaInputStream stream, ContentHandler handler, + Metadata metadata, ParseContext context, + boolean outputHtml) throws IOException { + throw new IOException("synthetic embedded parse failure"); + } + }); + } + + @Test + public void testEncryptedObjectGroupAndMissingBlobReference() throws Exception { + ObjectGroupDataElementData encryptedData = new ObjectGroupDataElementData(); + ObjectGroupObjectDeclare declaration = new ObjectGroupObjectDeclare(); + declaration.objectPartitionID.setDecodedValue(1); + ObjectGroupObjectData objectData = new ObjectGroupObjectData(); + objectData.data.content.add((byte) 42); + encryptedData.objectGroupDeclarations.objectDeclarationList.add(declaration); + encryptedData.objectGroupData.objectGroupObjectDataList.add(objectData); + + RevisionStoreObjectGroup encrypted = RevisionStoreObjectGroup.createInstance( + new ExGuid(3, UUID.randomUUID()), encryptedData, true); + assertEquals(1, encrypted.encryptionObjects.size()); + + ObjectGroupDataElementData missingBlobData = new ObjectGroupDataElementData(); + ObjectGroupObjectBLOBDataDeclaration blobDeclaration = + new ObjectGroupObjectBLOBDataDeclaration(); + blobDeclaration.objectExGUID = new ExGuid(4, UUID.randomUUID()); + blobDeclaration.objectPartitionID.setDecodedValue(2); + ObjectGroupObjectDataBLOBReference reference = + new ObjectGroupObjectDataBLOBReference(); + reference.blobExtendedGUID = new ExGuid(5, UUID.randomUUID()); + missingBlobData.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList + .add(blobDeclaration); + missingBlobData.objectGroupData.objectGroupObjectDataBLOBReferenceList.add(reference); + RevisionStoreObjectGroup missing = RevisionStoreObjectGroup.createInstance( + new ExGuid(6, UUID.randomUUID()), missingBlobData, false); + assertEquals(1, missing.objects.size()); + assertNull(missing.objects.get(0).fileDataObject.getData()); + } + + private MSOneStorePackage packageWithFileData(ObjectDataBLOBDataElementData blobData) { + DataElement element = new DataElement(); + element.data = blobData; + FileDataObject fileData = new FileDataObject(); + fileData.objectDataBLOBDataElement = element; + RevisionStoreObject object = new RevisionStoreObject(); + object.objectID = new ExGuid(7, UUID.randomUUID()); + object.fileDataObject = fileData; + RevisionStoreObjectGroup group = new RevisionStoreObjectGroup( + new ExGuid(8, UUID.randomUUID())); + group.objects.add(object); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.OtherFileNodeList.add(group); + return pkg; + } + + private void walkWithExtractor(MSOneStorePackage pkg, EmbeddedDocumentExtractor extractor) + throws Exception { + ParseContext context = new ParseContext(); + context.set(EmbeddedDocumentExtractor.class, extractor); + Metadata metadata = new Metadata(); + XHTMLContentHandler xhtml = new XHTMLContentHandler( + new ToTextContentHandler(new StringWriter()), metadata, context); + xhtml.startDocument(); + pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, xhtml, context); + xhtml.endDocument(); + } + + private DataElement roundTripBlobElement() throws Exception { + ObjectDataBLOBDataElementData blobData = new ObjectDataBLOBDataElementData(); + for (byte b : BLOB_BYTES) { + blobData.objectDataBLOB.data.content.add(b); + } + DataElement blobElement = + new DataElement(DataElementType.ObjectDataBLOBDataElementData, blobData); + byte[] serialized = ByteUtil.toByteArray(blobElement.serializeToByteList()); + return StreamObject.getCurrent(serialized, new AtomicInteger(0), DataElement.class); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java new file mode 100644 index 00000000000..e164455dab8 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.ArrayNumber; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.IProperty; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.NoData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtArrayOfPropertyValues; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtFourBytesOfLengthFollowedByData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySet; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySetObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestRootDeclare; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreCell; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObjectGroup; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellID; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyID; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyType; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectPropSet; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil; +import org.apache.tika.sax.ToTextContentHandler; +import org.apache.tika.sax.XHTMLContentHandler; + +public class MSOneStorePackageTest { + + @Test + public void testPagesFollowSectionOrderAndDropOlderCellVersions() throws Exception { + ExGuid sectionRootId = id(1); + CellID pageOne = cell(10, 100); + CellID pageTwo = cell(20, 200); + CellID oldPageOne = cell(11, 100); + RevisionStoreCell pageTwoCell = cellWithText(pageTwo, "page two"); + RevisionStoreCell oldPageOneCell = cellWithText(oldPageOne, "old page one"); + RevisionStoreCell pageOneCell = cellWithText(pageOne, "page one"); + RevisionStoreCell unrelatedCell = cellWithText(cell(30, 300), "unrelated"); + + RevisionStoreObject sectionRoot = object(sectionRootId, + propertySet(new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78, + new NoData()), + new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D79, + new NoData())), + Collections.emptyList(), Arrays.asList(pageOne, pageTwo)); + RevisionStoreCell section = new RevisionStoreCell(); + section.objectGroups.add(group(sectionRoot)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = sectionRootId; + section.rootDeclares.add(rootDeclare); + RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare(); + missingRoot.objectExGuid = id(1000); + section.rootDeclares.add(missingRoot); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.dataRootCell = section; + pkg.cells.addAll(Arrays.asList(pageTwoCell, oldPageOneCell, pageOneCell, unrelatedCell)); + + String text = walk(pkg); + assertTrue(text.indexOf("page one") < text.indexOf("page two")); + assertFalse(text.contains("old page one")); + assertTrue(text.contains("unrelated")); + } + + @Test + public void testUnresolvedRootsFallBackToAllObjects() throws Exception { + RevisionStoreCell cell = cellWithText(cell(1, 1), "fallback content"); + cell.rootDeclares.clear(); + RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare(); + missingRoot.objectExGuid = id(999); + cell.rootDeclares.add(missingRoot); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + + assertTrue(walk(pkg).contains("fallback content")); + } + + @Test + public void testPrimaryPictureSuppressesDerivedPicture() throws Exception { + ExGuid pictureID = id(40); + ExGuid webPictureID = id(41); + RevisionStoreObject root = object(id(42), propertySet( + new PropertySpec(PropertyType.ObjectID, 0x20001C3F, new NoData()), + new PropertySpec(PropertyType.ObjectID, 0x200034C8, new NoData())), + Arrays.asList(pictureID, webPictureID), Collections.emptyList()); + RevisionStoreObject picture = object(pictureID, propertySet(), + Collections.emptyList(), Collections.emptyList()); + picture.propertySet.objectSpaceObjectPropSet.body = null; + RevisionStoreObject webPicture = object(webPictureID, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("derived picture"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, picture, webPicture)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = root.objectID; + cell.rootDeclares.add(rootDeclare); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + + assertFalse(walk(pkg).contains("derived picture")); + } + + @Test + public void testNestedPropertySetsAndMissingReferencesAreTraversedSafely() throws Exception { + ExGuid childId = id(2); + PropertySet nested = propertySet(new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("nested text"))); + PrtArrayOfPropertyValues array = new PrtArrayOfPropertyValues(); + array.data = new PropertySet[]{propertySet(new PropertySpec( + PropertyType.FourBytesOfLengthFollowedByData, 0x1C003498, text("array text")))}; + RevisionStoreObject root = object(id(1), propertySet( + new PropertySpec(PropertyType.PropertySet, 0, nested), + new PropertySpec(PropertyType.ArrayOfPropertyValues, 0, array), + new PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001D5F, + arrayNumber(2)), + new PropertySpec(PropertyType.ObjectID, 0x24001D5F, new NoData()), + new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78, new NoData()), + new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D79, new NoData()), + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C001DD7, bytes((byte) 'u', (byte) 0, (byte) 1)), + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C001C22, bytes((byte) 'h', (byte) 0, (byte) 'i', (byte) 0, + (byte) 0, (byte) 0))), + Collections.singletonList(childId), Collections.singletonList(cell(50, 51))); + RevisionStoreObject child = object(childId, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("child text"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, child)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = root.objectID; + cell.rootDeclares.add(rootDeclare); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + String text = walk(pkg); + assertTrue(text.contains("nested text")); + assertTrue(text.contains("array text")); + assertTrue(text.contains("child text")); + assertTrue(text.contains("u")); + assertTrue(text.contains("hi")); + } + + private static String walk(MSOneStorePackage pkg) throws Exception { + Metadata metadata = new Metadata(); + StringWriter writer = new StringWriter(); + XHTMLContentHandler xhtml = new XHTMLContentHandler( + new ToTextContentHandler(writer), metadata, new ParseContext()); + xhtml.startDocument(); + pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, xhtml); + xhtml.endDocument(); + return writer.toString(); + } + + private static RevisionStoreCell cellWithText(CellID cellID, String value) throws Exception { + RevisionStoreObject object = object(id(cellID.extendGUID1.hashCode()), + propertySet(new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text(value))), Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.cellID = cellID; + cell.objectGroups.add(group(object)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = object.objectID; + cell.rootDeclares.add(rootDeclare); + return cell; + } + + private static RevisionStoreObject object(ExGuid objectID, PropertySet body, + List references, List spaces) + throws Exception { + RevisionStoreObject object = new RevisionStoreObject(); + object.objectID = objectID; + PropertySetObject propertySetObject = new PropertySetObject(null, emptyObjectData()); + ObjectSpaceObjectPropSet propSet = new ObjectSpaceObjectPropSet(); + propSet.body = body; + propertySetObject.objectSpaceObjectPropSet = propSet; + object.propertySet = propertySetObject; + if (!references.isEmpty()) { + object.referencedObjectID = new org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGUIDArray(); + object.referencedObjectID.content = references; + } + if (!spaces.isEmpty()) { + object.referencedObjectSpacesID = new org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellIDArray(); + object.referencedObjectSpacesID.content = spaces; + } + return object; + } + + private static RevisionStoreObjectGroup group(RevisionStoreObject... objects) { + RevisionStoreObjectGroup group = new RevisionStoreObjectGroup(id(500)); + group.objects.addAll(Arrays.asList(objects)); + return group; + } + + private static PropertySet propertySet(PropertySpec... specs) { + PropertySet set = new PropertySet(); + set.cProperties = specs.length; + set.rgPrids = new PropertyID[specs.length]; + set.rgData = new ArrayList<>(); + for (int i = 0; i < specs.length; i++) { + set.rgPrids[i] = propertyID(specs[i].type, specs[i].value); + set.rgData.add(specs[i].property); + } + return set; + } + + private static PropertyID propertyID(PropertyType type, int value) { + PropertyID id = new PropertyID(); + id.type = type.getIntVal(); + id.value = value; + return id; + } + + private static PrtFourBytesOfLengthFollowedByData bytes(byte... value) { + PrtFourBytesOfLengthFollowedByData data = new PrtFourBytesOfLengthFollowedByData(); + data.data = value; + data.cb = data.data.length; + return data; + } + + private static PrtFourBytesOfLengthFollowedByData text(String value) { + PrtFourBytesOfLengthFollowedByData data = new PrtFourBytesOfLengthFollowedByData(); + data.data = value.getBytes(StandardCharsets.US_ASCII); + data.cb = data.data.length; + return data; + } + + private static ArrayNumber arrayNumber(int number) { + ArrayNumber array = new ArrayNumber(); + array.number = number; + return array; + } + + private static ObjectGroupObjectData emptyObjectData() { + ObjectGroupObjectData data = new ObjectGroupObjectData(); + data.data.content.addAll(ByteUtil.toListOfByte(new byte[]{0, 0, 0, (byte) 0x80, + 0, 0, 0, 0})); + return data; + } + + private static CellID cell(int first, int second) { + return new CellID(id(first), id(second)); + } + + private static ExGuid id(int value) { + return new ExGuid(value, UUID.nameUUIDFromBytes(("id-" + value).getBytes(StandardCharsets.UTF_8))); + } + + private static final class PropertySpec { + private final PropertyType type; + private final int value; + private final IProperty property; + + private PropertySpec(PropertyType type, int value, IProperty property) { + this.type = type; + this.value = value; + this.property = property; + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParserTest.java new file mode 100644 index 00000000000..e576de2b9be --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParserTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.UUID; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.CellManifestCurrentRevision; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.CellManifestDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.DataElement; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestObjectGroupReferences; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreCell; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StorageIndexCellMapping; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StorageIndexDataElementData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.StorageIndexRevisionMapping; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellID; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid; + +public class MSOneStoreParserTest { + + @Test + public void testMissingRootsAndRevisionMappingsReturnNoCell() throws Exception { + MSOneStoreParser parser = new MSOneStoreParser(); + set(parser, "cellManifestDataElements", new java.util.ArrayList<>()); + set(parser, "revisionManifestDataElements", new java.util.ArrayList<>()); + set(parser, "objectGroupDataElements", new java.util.ArrayList<>()); + set(parser, "objectBlOBElementsById", new java.util.HashMap<>()); + MSOneStorePackage pkg = new MSOneStorePackage(); + CellID cellID = cell(1); + + assertNull(parseCell(parser, cellID, pkg)); + + pkg.storageIndex = new StorageIndexDataElementData(); + StorageIndexCellMapping cellMapping = new StorageIndexCellMapping(); + cellMapping.cellID = cellID; + cellMapping.cellMappingExGuid = id(2); + pkg.storageIndex.storageIndexCellMappingList.add(cellMapping); + assertNull(parseCell(parser, cellID, pkg)); + + DataElement cellManifestElement = new DataElement(); + cellManifestElement.dataElementExGuid = cellMapping.cellMappingExGuid; + cellManifestElement.data = new CellManifestDataElementData(); + set(parser, "cellManifestDataElements", Arrays.asList(cellManifestElement)); + assertNull(parseCell(parser, cellID, pkg)); + + CellManifestDataElementData cellManifest = (CellManifestDataElementData) + cellManifestElement.data; + cellManifest.cellManifestCurrentRevision = null; + assertNull(parseCell(parser, cellID, pkg)); + cellManifest.cellManifestCurrentRevision = new CellManifestCurrentRevision(); + ExGuid missingRevisionID = id(3); + cellManifest.cellManifestCurrentRevision.cellManifestCurrentRevisionExGuid = + missingRevisionID; + StorageIndexRevisionMapping missingManifestMapping = new StorageIndexRevisionMapping(); + missingManifestMapping.revisionExGuid = missingRevisionID; + missingManifestMapping.revisionMappingExGuid = id(4); + pkg.storageIndex.storageIndexRevisionMappingList.add(missingManifestMapping); + assertNull(parseCell(parser, cellID, pkg)); + } + + @Test + public void testRevisionChainStopsCyclesAndDeduplicatesObjectGroups() throws Exception { + MSOneStoreParser parser = new MSOneStoreParser(); + set(parser, "cellManifestDataElements", new java.util.ArrayList<>()); + set(parser, "revisionManifestDataElements", new java.util.ArrayList<>()); + set(parser, "objectGroupDataElements", new java.util.ArrayList<>()); + set(parser, "objectBlOBElementsById", new java.util.HashMap<>()); + + CellID cellID = cell(20); + ExGuid cellMappingID = id(21); + ExGuid currentMappingID = id(22); + ExGuid oldMappingID = id(23); + ExGuid currentRevisionID = id(24); + ExGuid oldRevisionID = id(25); + ExGuid currentGroupID = id(26); + ExGuid oldGroupID = id(27); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.storageIndex = new StorageIndexDataElementData(); + StorageIndexCellMapping cellMapping = new StorageIndexCellMapping(); + cellMapping.cellID = cellID; + cellMapping.cellMappingExGuid = cellMappingID; + pkg.storageIndex.storageIndexCellMappingList.add(cellMapping); + StorageIndexRevisionMapping currentMapping = new StorageIndexRevisionMapping(); + currentMapping.revisionMappingExGuid = currentMappingID; + currentMapping.revisionExGuid = currentRevisionID; + StorageIndexRevisionMapping oldMapping = new StorageIndexRevisionMapping(); + oldMapping.revisionMappingExGuid = oldMappingID; + oldMapping.revisionExGuid = oldRevisionID; + pkg.storageIndex.storageIndexRevisionMappingList.add(currentMapping); + pkg.storageIndex.storageIndexRevisionMappingList.add(oldMapping); + + DataElement cellManifestElement = new DataElement(); + cellManifestElement.dataElementExGuid = cellMappingID; + CellManifestDataElementData cellManifest = new CellManifestDataElementData(); + cellManifest.cellManifestCurrentRevision.cellManifestCurrentRevisionExGuid = currentRevisionID; + cellManifestElement.data = cellManifest; + set(parser, "cellManifestDataElements", Arrays.asList(cellManifestElement)); + + RevisionManifestDataElementData current = revision(currentRevisionID, oldRevisionID, + currentGroupID, currentGroupID, id(99)); + RevisionManifestDataElementData old = revision(oldRevisionID, currentRevisionID, + oldGroupID); + DataElement currentElement = new DataElement(); + currentElement.dataElementExGuid = currentMappingID; + currentElement.data = current; + DataElement oldElement = new DataElement(); + oldElement.dataElementExGuid = oldMappingID; + oldElement.data = old; + set(parser, "revisionManifestDataElements", Arrays.asList(currentElement, oldElement)); + + DataElement currentGroup = new DataElement(); + currentGroup.dataElementExGuid = currentGroupID; + currentGroup.data = new ObjectGroupDataElementData(); + DataElement oldGroup = new DataElement(); + oldGroup.dataElementExGuid = oldGroupID; + oldGroup.data = new ObjectGroupDataElementData(); + set(parser, "objectGroupDataElements", Arrays.asList(currentGroup, oldGroup)); + + RevisionStoreCell result = parseCell(parser, cellID, pkg); + assertEquals(2, result.objectGroups.size()); + + old.revisionManifest.baseRevisionID = id(1001); + RevisionStoreCell missingBaseResult = parseCell(parser, cellID, pkg); + assertEquals(2, missingBaseResult.objectGroups.size()); + } + + private static RevisionManifestDataElementData revision(ExGuid revisionID, + ExGuid baseRevisionID, + ExGuid... groups) { + RevisionManifestDataElementData data = new RevisionManifestDataElementData(); + data.revisionManifest.revisionID = revisionID; + data.revisionManifest.baseRevisionID = baseRevisionID; + for (ExGuid group : groups) { + RevisionManifestObjectGroupReferences reference = + new RevisionManifestObjectGroupReferences(); + reference.objectGroupExtendedGUID = group; + data.revisionManifestObjectGroupReferences.add(reference); + } + return data; + } + + private static RevisionStoreCell parseCell(MSOneStoreParser parser, CellID cellID, + MSOneStorePackage pkg) throws Exception { + Method method = MSOneStoreParser.class.getDeclaredMethod("parseCell", CellID.class, + MSOneStorePackage.class); + method.setAccessible(true); + return (RevisionStoreCell) method.invoke(parser, cellID, pkg); + } + + private static void set(MSOneStoreParser parser, String field, Object value) throws Exception { + java.lang.reflect.Field declared = MSOneStoreParser.class.getDeclaredField(field); + declared.setAccessible(true); + declared.set(parser, value); + } + + private static CellID cell(int value) { + return new CellID(id(value), id(value + 1)); + } + + private static ExGuid id(int value) { + return new ExGuid(value, UUID.nameUUIDFromBytes( + ("parser-" + value).getBytes(StandardCharsets.UTF_8))); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNoteEmbeddedImage.one b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNoteEmbeddedImage.one new file mode 100644 index 00000000000..2e2b1229525 Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testOneNoteEmbeddedImage.one differ