Fix TimeGenerated in azure log analytics store

Conversion to UTC
Always remove the TZ
Better millisecond rounding

Author: np5

tests/stores/test_azure_log_analytics.py

@@ -0,0 +1,27 @@
+from dateutil import parser
+from django.test import SimpleTestCase
+from zentral.core.stores.backends.azure_log_analytics import datetime_to_iso8601z_truncated_to_milliseconds
+
+
+class TestDateTimeConverstion(SimpleTestCase):
+    def _assert_ts_equals(self, ts_list):
+        for in_ts, out_ts in ts_list:
+            self.assertEqual(
+                datetime_to_iso8601z_truncated_to_milliseconds(parser.parse(in_ts)),
+                out_ts
+            )
+
+    def test_timezone_conversion(self):
+        self._assert_ts_equals((
+            ("2019-01-12T11:11:11.319+02:00", "2019-01-12T09:11:11.319Z"),
+            ("2019-01-12T11:11:11.319+00:00", "2019-01-12T11:11:11.319Z"),
+            ("2019-01-12T11:11:11Z", "2019-01-12T11:11:11Z"),
+        ))
+
+    def test_microseconds_to_milliseconds(self):
+        self._assert_ts_equals((
+            ("2019-01-12T11:11:11.999999", "2019-01-12T11:11:12Z"),
+            ("2019-01-12T11:11:11.123111", "2019-01-12T11:11:11.123Z"),
+            ("2019-01-12T11:11:11.000999+00:00", "2019-01-12T11:11:11.001Z"),
+            ("2019-01-12T11:11:11.000234+00:00", "2019-01-12T11:11:11Z"),
+        ))

zentral/core/stores/backends/azure_log_analytics.py

@@ -1,10 +1,11 @@
 import base64
 import collections.abc
-from datetime import datetime
+from datetime import datetime, timedelta
 import hashlib
 import hmac
 import json
 import logging
+import pytz
 import requests
 from zentral.core.events import event_from_event_d
 from zentral.core.stores.backends.base import BaseEventStore
@@ -13,6 +14,28 @@
 logger = logging.getLogger('zentral.core.stroes.backends.azure_log_analytics')
 
 
+def datetime_to_iso8601z_truncated_to_milliseconds(dt):
+    # round created at to milliseconds
+    dt_microsecond = dt.microsecond
+    if dt_microsecond:
+        dt_millisecond = round(dt_microsecond / 1000)
+        if dt_millisecond == 1000:
+            dt = dt.replace(microsecond=0)
+            dt += timedelta(seconds=1)
+        else:
+            dt = dt.replace(microsecond=1000 * dt_millisecond)
+
+    # convert created at to UTC, remove the TZ info (naive datetime), convert to isoformat
+    dt_iso = dt.astimezone(pytz.utc).replace(tzinfo=None).isoformat()
+
+    # truncate the microseconds in isoformat if necessary
+    if "." in dt_iso:
+        dt_iso = dt_iso[:-3]
+
+    # add the pseudo time zone
+    return "{}Z".format(dt_iso)
+
+
 class EventStore(BaseEventStore):
     log_type = "ZentralEvent"
     content_type = "application/json"
@@ -53,23 +76,8 @@ def _prepare_event(self, event):
 
         metadata = event_d.pop("_zentral")
 
-        # created at
-        created_at = metadata.pop("created_at")
-        if "." in created_at:
-            created_at, created_at_ms = created_at.split(".")
-            if len(created_at_ms) == 6:
-                created_at_ms = round(int(created_at_ms) / 1000)
-            elif len(created_at_ms) == 3:
-                created_at_ms = int(created_at_ms)
-            else:
-                # TODO
-                created_at_ms = 0
-            if created_at_ms:
-                if created_at_ms == 1000:
-                    # TODO
-                    created_at_ms = 999
-                created_at = "{}.{:03d}".format(created_at, created_at_ms)
-        metadata["created_at"] = "{}Z".format(created_at)
+        # fix created_at format for use as TimeGenerated field via the time-generated-field header
+        metadata["created_at"] = datetime_to_iso8601z_truncated_to_milliseconds(event.metadata.created_at)
 
         # flatten the metadata
         azure_event = self._flatten_metadata(metadata)