TIKA-4806 -- avoid overwriting dc terms with mapi properties that aren't a great fit (#2988)

Author: tballison

tika-core/src/main/java/org/apache/tika/metadata/MAPI.java

@@ -52,6 +52,13 @@ public interface MAPI {
     // PidTagClientSubmitTime -- distinct MAPI prop from the provider-side SUBMISSION_ACCEPTED_AT_TIME.
     Property CLIENT_SUBMIT_TIME = Property.internalDate(PREFIX_MAPI_META + "msg-client-submit-time");
 
+    // CREATED/MODIFIED fallback when no CLIENT_SUBMIT_TIME.
+    Property MESSAGE_DELIVERY_TIME = Property.internalDate(PREFIX_MAPI_META + "message-delivery-time");
+
+    // Storage-level timestamps, not content dates -- don't use for CREATED/MODIFIED (TIKA-4798).
+    Property CREATION_TIME = Property.internalDate(PREFIX_MAPI_META + "creation-time");
+    Property LAST_MODIFICATION_TIME = Property.internalDate(PREFIX_MAPI_META + "last-modification-time");
+
     Property SUBMISSION_ID = Property.internalText(PREFIX_MAPI_META + "msg-submission-id");
 
     Property INTERNET_MESSAGE_ID = Property.internalText(PREFIX_MAPI_META + "internet-message-id");

tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json

@@ -189,15 +189,18 @@
   {"class":"org.apache.tika.metadata.MAPI","field":"CLIENT_SUBMIT_TIME","key":"mapi:msg-client-submit-time"},
   {"class":"org.apache.tika.metadata.MAPI","field":"CONVERSATION_INDEX","key":"mapi:conversation-index"},
   {"class":"org.apache.tika.metadata.MAPI","field":"CONVERSATION_TOPIC","key":"mapi:conversation-topic"},
+  {"class":"org.apache.tika.metadata.MAPI","field":"CREATION_TIME","key":"mapi:creation-time"},
   {"class":"org.apache.tika.metadata.MAPI","field":"FROM_REPRESENTING_EMAIL","key":"mapi:from-representing-email"},
   {"class":"org.apache.tika.metadata.MAPI","field":"FROM_REPRESENTING_NAME","key":"mapi:from-representing-name"},
   {"class":"org.apache.tika.metadata.MAPI","field":"IMPORTANCE","key":"mapi:importance"},
   {"class":"org.apache.tika.metadata.MAPI","field":"INTERNET_MESSAGE_ID","key":"mapi:internet-message-id"},
   {"class":"org.apache.tika.metadata.MAPI","field":"INTERNET_REFERENCES","key":"mapi:internet-references"},
   {"class":"org.apache.tika.metadata.MAPI","field":"IN_REPLY_TO_ID","key":"mapi:in-reply-to-id"},
   {"class":"org.apache.tika.metadata.MAPI","field":"IS_FLAGGED","key":"mapi:is-flagged"},
+  {"class":"org.apache.tika.metadata.MAPI","field":"LAST_MODIFICATION_TIME","key":"mapi:last-modification-time"},
   {"class":"org.apache.tika.metadata.MAPI","field":"MESSAGE_CLASS","key":"mapi:message-class"},
   {"class":"org.apache.tika.metadata.MAPI","field":"MESSAGE_CLASS_RAW","key":"mapi:message-class-raw"},
+  {"class":"org.apache.tika.metadata.MAPI","field":"MESSAGE_DELIVERY_TIME","key":"mapi:message-delivery-time"},
   {"class":"org.apache.tika.metadata.MAPI","field":"PRIORTY","key":"mapi:priority"},
   {"class":"org.apache.tika.metadata.MAPI","field":"RECIPIENTS_STRING","key":"mapi:recipients-string"},
   {"class":"org.apache.tika.metadata.MAPI","field":"SENT_BY_SERVER_TYPE","key":"mapi:sent-by-server-type"},

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/pst/PSTMailItemParser.java

@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Date;
 import java.util.Set;
 
 import com.pff.PSTAttachment;
@@ -138,9 +139,17 @@ private void extractMetadata(PSTMessage pstMail, Metadata metadata) {
         metadata.set(TikaCoreProperties.SUBJECT, pstMail.getSubject());
         metadata.set(Metadata.MESSAGE_FROM, pstMail.getSenderName());
         metadata.set(TikaCoreProperties.CREATOR, pstMail.getSenderName());
-        metadata.set(TikaCoreProperties.CREATED, pstMail.getCreationTime());
+        //creation/last-modification are storage timestamps, not content dates (TIKA-4798)
+        Date messageDate = pstMail.getClientSubmitTime();
+        if (messageDate == null) {
+            messageDate = pstMail.getMessageDeliveryTime();
+        }
+        metadata.set(TikaCoreProperties.CREATED, messageDate);
+        metadata.set(TikaCoreProperties.MODIFIED, messageDate);
         metadata.set(MAPI.CLIENT_SUBMIT_TIME, pstMail.getClientSubmitTime());
-        metadata.set(TikaCoreProperties.MODIFIED, pstMail.getLastModificationTime());
+        metadata.set(MAPI.MESSAGE_DELIVERY_TIME, pstMail.getMessageDeliveryTime());
+        metadata.set(MAPI.CREATION_TIME, pstMail.getCreationTime());
+        metadata.set(MAPI.LAST_MODIFICATION_TIME, pstMail.getLastModificationTime());
         metadata.set(TikaCoreProperties.COMMENTS, pstMail.getComment());
         metadata.set(PST.DESCRIPTOR_NODE_ID, valueOf(pstMail.getDescriptorNodeId()));
         metadata.set(Message.MESSAGE_FROM_EMAIL, pstMail.getSenderEmailAddress());

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/pst/OutlookPSTParserTest.java

@@ -79,6 +79,14 @@ public void testExtendedMetadata() throws Exception {
         assertEquals("Jörn Kottmann", m1.get(MAPI.FROM_REPRESENTING_NAME));
         assertEquals("kottmann@gmail.com", m1.get(MAPI.FROM_REPRESENTING_EMAIL));
         assertEquals("NOTE", m1.get(MAPI.MESSAGE_CLASS));
+        //TIKA-4806: CREATED/MODIFIED reflect the client submit time, not the storage-level
+        //PidTagLastModificationTime (2014-02-26T12:37:43Z), which is ~5 hours later here.
+        assertEquals("2014-02-26T07:50:04Z", m1.get(TikaCoreProperties.CREATED));
+        assertEquals("2014-02-26T07:50:04Z", m1.get(TikaCoreProperties.MODIFIED));
+        assertEquals("2014-02-26T07:50:04Z", m1.get(MAPI.CLIENT_SUBMIT_TIME));
+        assertEquals("2014-02-26T07:51:02Z", m1.get(MAPI.CREATION_TIME));
+        assertEquals("2014-02-26T12:37:43Z", m1.get(MAPI.LAST_MODIFICATION_TIME));
+        assertEquals("2014-02-26T07:51:02Z", m1.get(MAPI.MESSAGE_DELIVERY_TIME));
         assertEquals("/Début du fichier de données Outlook/Re: Feature Generators.msg",
                 m1.get(TikaCoreProperties.INTERNAL_PATH));
         //test that subject is making it into the xhtml