Fixing UTC metadata not showing for other endpoints

Author: huzaifams-svg

api/datalake_api/fetcher.py

@@ -77,20 +77,6 @@ def get_file(self, file_id):
         key = self._get_s3_key(file_id)
         fd = key['Body']
         j = json.loads(key['Metadata']['datalake'])
-        start_utc = j['start']
-        end_utc = j['end']
-
-        if start_utc:
-            start_utc = datetime.fromtimestamp(
-                start_utc / 1000.0, tz=timezone.utc
-            ).isoformat()
-        if end_utc:
-            end_utc = datetime.fromtimestamp(
-                end_utc / 1000.0, tz=timezone.utc
-            ).isoformat()
-        j['start_UTC'] = start_utc
-        j['end_UTC'] = end_utc
-
         metadata = Metadata(j)
         return ArchiveFile(fd, metadata)
 

api/datalake_api/v0.py

@@ -18,6 +18,8 @@
 from flask import current_app as app
 import os
 import simplejson as json
+from datetime import datetime, timezone
+import decimal
 from .querier import ArchiveQuerier, Cursor, InvalidCursor, \
     DEFAULT_LOOKBACK_DAYS
 from .fetcher import ArchiveFileFetcher
@@ -29,6 +31,36 @@
 
 _archive_querier = None
 
+def add_utc_metadata(metadata):
+    """Add UTC timestamp fields to metadata
+
+    This function takes a metadata dict and adds start_UTC and end_UTC fields
+    containing ISO-formatted UTC timestamps corresponding to the millisecond
+    timestamps in the start and end fields.
+    Can be expanded to add any needed metadata to apis
+    """
+    if not metadata:
+        return metadata
+
+    start_utc = metadata['start']
+    end_utc = metadata['end']
+    if start_utc:
+        if isinstance(start_utc, decimal.Decimal):
+            start_utc = float(start_utc)
+        start_utc = datetime.fromtimestamp(
+            start_utc / 1000.0, tz=timezone.utc
+        ).isoformat()
+    if end_utc:
+        if isinstance(end_utc, decimal.Decimal):
+            end_utc = float(end_utc)
+        end_utc = datetime.fromtimestamp(
+            end_utc / 1000.0, tz=timezone.utc
+        ).isoformat()
+
+    metadata['start_UTC'] = start_utc
+    metadata['end_UTC'] = end_utc
+    return metadata
+
 def _get_aws_kwargs():
     kwargs = dict(
         region_name=app.config.get('AWS_REGION'),
@@ -349,7 +381,11 @@ def files_get():
                                    where=params.get('where'),
                                    cursor=params.get('cursor'))
 
-    [r.update(http_url=_get_canonical_http_url(r)) for r in results]
+    for r in results:
+        r.update(http_url=_get_canonical_http_url(r))
+        # Add UTC timestamps to each record's metadata
+        r['metadata'] = add_utc_metadata(r['metadata'])
+
     response = {
         'records': results,
         'next': _get_next_url(flask.request, results),
@@ -476,6 +512,7 @@ def file_get_metadata(file_id):
           id: DatalakeAPIError
     '''
     f = _get_file(file_id)
+    f.metadata = add_utc_metadata(f.metadata)
     return Response(json.dumps(f.metadata), content_type='application/json')
 
 
@@ -542,6 +579,8 @@ def latest_get(what, where):
     params = _validate_latest_params(params)
     f = _get_latest(what, where, params.get('lookback', DEFAULT_LOOKBACK_DAYS))
     f.update(http_url=_get_canonical_http_url(f))
+    # Add UTC timestamps to metadata
+    f['metadata'] = add_utc_metadata(f['metadata'])
     return Response(json.dumps(f), content_type='application/json')