Pclean

pipeline/domain/measurementset.py

@@ -19,7 +19,8 @@
 from pipeline.infrastructure.launcher import current_task_name
 
 if TYPE_CHECKING:
-    from pipeline.domain import Antenna, AntennaArray, DataDescription, DataType, Field, Polarization, Scan, State
+    from pipeline.domain import (Antenna, AntennaArray, DataDescription,
+                                 DataType, Field, Polarization, Scan, State)
     from pipeline.infrastructure.tablereader import RetrieveByIndexContainer
 
 LOG = infrastructure.logging.get_logger(__name__)
@@ -885,8 +886,8 @@ def get_alma_cycle_number(self) -> int | None:
             return None
         start_time = utils.get_epoch_as_datetime(self.start_time)
         for cycle, (start_str, end_str) in cycle_numbers.items():
-            start = datetime.datetime.strptime(start_str, '%Y-%m-%d')
-            end = datetime.datetime.strptime(end_str, '%Y-%m-%d')
+            start = datetime.datetime.strptime(start_str, '%Y-%m-%d').replace(tzinfo=datetime.timezone.utc)
+            end = datetime.datetime.strptime(end_str, '%Y-%m-%d').replace(tzinfo=datetime.timezone.utc)
             if start <= start_time <= end:
                 return cycle
         return None  # No match

pipeline/domain/measures.py

@@ -1118,7 +1118,7 @@ class TimeInterval(object):
     """
     __slots__ = ('start', 'end')
 
-    FOREVER = datetime.datetime(9999, 12, 31)
+    FOREVER = datetime.datetime(9999, 12, 31, tzinfo=datetime.timezone.utc)
 
     def __getstate__(self) -> tuple[datetime.datetime, datetime.datetime]:
         return self.start, self.end
@@ -1218,4 +1218,4 @@ def starting_from_now(cls) -> TimeInterval:
         Returns:
             TimeInterval object from "utcnow()" until 31-12-9999.
         """
-        return cls(datetime.datetime.utcnow(), cls.FOREVER)
+        return cls(datetime.datetime.now(datetime.timezone.utc), cls.FOREVER)

pipeline/domain/state.py

@@ -365,7 +365,7 @@ def __init__(self, state_id: int, obs_mode: str) -> None:
         super(StateNAOJ, self).__init__(state_id, obs_mode)
 
 
-class StateFactory(object):
+class StateFactory:
     """
     Factory class to create State objects based on given observatory and
     observation/measurement set start time.
@@ -380,7 +380,7 @@ def __init__(self, observatory: str, start: datetime.datetime | None = None) ->
                 to distinguish between Cycle 0 and later ALMA datasets.
         """
         if observatory == 'ALMA':
-            if start and start < datetime.datetime(2013, 1, 21):
+            if start and start < datetime.datetime(2013, 1, 21, tzinfo=datetime.timezone.utc):
                 self._constructor = StateALMACycle0
             else:
                 self._constructor = StateALMA

pipeline/h/tasks/exportdata/aqua.py

@@ -167,7 +167,7 @@ def get_qa_summary(self, context):
         root = ElementTree.Element('QaSummary')
 
         # Generate the report date
-        now = datetime.datetime.utcnow()
+        now = datetime.datetime.now(datetime.timezone.utc)
         ElementTree.SubElement(root, 'ReportDate').text = now.strftime('%Y-%m-%d %H:%M:%S')
 
         # Processing time

pipeline/hifa/tasks/importdata/dbfluxes.py

@@ -255,7 +255,7 @@ def add_catalogue_fluxes(
     science_windows = ms.get_spectral_windows(science_windows_only=True)
 
     # Test query to see if we need to switch to the backup URL
-    obs_time = datetime.datetime(2013, 3, 27, 7, 53, 3, 168000)
+    obs_time = datetime.datetime(2013, 3, 27, 7, 53, 3, 168000, tzinfo=datetime.timezone.utc)
     freq_hz = '86837309056.169219970703125'
     source_name = 'J1427-4206'
     contact_fail = False

pipeline/hifv/tasks/flagging/checkflag.py

@@ -191,7 +191,7 @@ def prepare(self):
             # plots['before'] = self._create_summaryplots(suffix='before', plotms_args=plot_selectdata)
 
         # PIPE-987: backup flagversion before rfi flagging
-        now_str = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S")
+        now_str = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S")
         job = casa_tasks.flagmanager(vis=self.inputs.vis, mode='save',
                                      versionname='hifv_checkflag_{}_stage{}_{}'.format(
                                          self.inputs.checkflagmode, self.inputs.context.task_counter, now_str),

pipeline/hifv/tasks/syspower/display.py

@@ -186,11 +186,17 @@ def create_plot(self, prefix):
             q1 = qa.quantity(time, 's')
             time1 = qa.time(q1, form='fits')
             try:
-                datetime_object = datetime.datetime.strptime(time1[0], '%Y-%m-%dT%H:%M:%S')
+                datetime_object = (
+                    datetime.datetime.fromisoformat(time1[0])
+                    .replace(tzinfo=datetime.timezone.utc)
+                )
             except ValueError:
                 timestr = time1[0]
                 timestr = timestr.replace('T24', 'T23')
-                datetime_object = datetime.datetime.strptime(timestr, '%Y-%m-%dT%H:%M:%S')
+                datetime_object = (
+                    datetime.datetime.fromisoformat(timestr)
+                    .replace(tzinfo=datetime.timezone.utc)
+                )
                 datetime_object += datetime.timedelta(hours=1)
             utc_time.append(datetime_object)
 
@@ -328,11 +334,17 @@ def create_plot(self, prefix):
             q1 = qa.quantity(time, 's')
             time1 = qa.time(q1, form='fits')
             try:
-                datetime_object = datetime.datetime.strptime(time1[0], '%Y-%m-%dT%H:%M:%S')
+                datetime_object = (
+                    datetime.datetime.fromisoformat(time1[0])
+                    .replace(tzinfo=datetime.timezone.utc)
+                )
             except ValueError:
                 timestr = time1[0]
                 timestr = timestr.replace('T24', 'T23')
-                datetime_object = datetime.datetime.strptime(timestr, '%Y-%m-%dT%H:%M:%S')
+                datetime_object = (
+                    datetime.datetime.fromisoformat(timestr)
+                    .replace(tzinfo=datetime.timezone.utc)
+                )
                 datetime_object += datetime.timedelta(hours=1)
             utc_time.append(datetime_object)
 

pipeline/infrastructure/basetask.py

@@ -38,9 +38,9 @@
 def timestamp(method):
     @functools.wraps(method)
     def attach_timestamp_to_results(self, *args, **kw):
-        start = datetime.datetime.utcnow()
+        start = datetime.datetime.now(datetime.timezone.utc)
         result = method(self, *args, **kw)
-        end = datetime.datetime.utcnow()
+        end = datetime.datetime.now(datetime.timezone.utc)
 
         if result is not None:
             result.timestamps = Timestamps(start, end)

pipeline/infrastructure/callibrary.py

@@ -28,7 +28,7 @@
 CalAppOrigin = collections.namedtuple('CalAppOrigin', ['task', 'inputs'])
 
 # observations before this date are considered Cycle 0 observations
-CYCLE_0_END_DATE = datetime.datetime(2013, 1, 21)
+CYCLE_0_END_DATE = datetime.datetime(2013, 1, 21, tzinfo=datetime.timezone.utc)
 
 
 class CalApplication(object):

pipeline/infrastructure/casa_tools.py

@@ -58,11 +58,11 @@ def format_arg_value(arg_val):
         tool_call = '{!s}.{!s}({!s})'.format(fn.__module__, fn.__name__, ', '.join(msg_args))
         CASACALLS_LOG.log(level, tool_call)
 
-        start_time = datetime.datetime.utcnow()
+        start_time = datetime.datetime.now(datetime.timezone.utc)
         try:
             return fn(*args, **kwargs)
         finally:
-            end_time = datetime.datetime.utcnow()
+            end_time = datetime.datetime.now(datetime.timezone.utc)
             elapsed = end_time - start_time
             LOG.log(level, '{} CASA tool call took {}s'.format(tool_call, elapsed.total_seconds()))
 

pipeline/infrastructure/launcher.py

@@ -119,7 +119,7 @@ def __init__(self, name: str | None = None) -> None:
         if name is None:
             # initialise the context name with something reasonable: a current
             # timestamp
-            now = datetime.datetime.utcnow()
+            now = datetime.datetime.now(datetime.timezone.utc)
             name = now.strftime('pipeline-%Y%m%dT%H%M%S')
         self.name = name
 

pipeline/infrastructure/pipeline_statistics.py

@@ -269,7 +269,7 @@ def stage_duration(context) -> list:
     timestamps = [r.read().timestamps.start for r in context.results]
 
     # tentative task end time stamp for the last stage
-    timestamps.append(datetime.datetime.utcnow())
+    timestamps.append(datetime.datetime.now(datetime.timezone.utc))
     task_duration = []
     for i in range(len(context.results)):
         # task execution duration

pipeline/infrastructure/renderer/htmlrenderer.py

@@ -791,7 +791,7 @@ def get_display_context(context):
         ## The end time of the last task is tentatively defined as the time of current time.
         timestamps = [ r.timestamps.start for r in context.results ]
         # tentative task end time stamp for the last stage
-        timestamps.append(datetime.datetime.utcnow())
+        timestamps.append(datetime.datetime.now(datetime.timezone.utc))
         task_duration = []
         for i in range(len(context.results)):
             # task execution duration

pipeline/infrastructure/renderer/logger.py

@@ -19,7 +19,7 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
 # *****************************************************************************
 import collections
-import distutils.spawn as spawn
+import shutil
 import itertools
 import os
 import platform
@@ -33,31 +33,29 @@
 LOG = logging.get_logger(__name__)
 
 # Set the command used to shrink plots down to thumbnails. If set to None, no
-# thumbnails will be generated 
-THUMBNAIL_CMD = None 
+# thumbnails will be generated
+THUMBNAIL_CMD = None
 
 
 # first try to find ImageMagick's 'mogrify' command. We assume that
-# ImageMagick's 'convert' commnand can be found in the same directory. We
+# ImageMagick's 'convert' command can be found in the same directory. We
 # do not search for 'convert' directly as some utilities also provide a
-# 'convert' command which may come earlier on the PATH. 
-mogrify_path = spawn.find_executable('mogrify')
+# 'convert' command which may come earlier on the PATH.
+mogrify_path = shutil.which('mogrify')
 if mogrify_path:
     bin_dir = os.path.dirname(mogrify_path)
     convert_path = os.path.join(bin_dir, 'convert')
     if os.path.exists(convert_path):
-        LOG.trace('Using convert executable at \'%s\' to generate '
-                  'thumbnails' % convert_path)
-        THUMBNAIL_CMD = lambda full, thumb : (convert_path, full, '-thumbnail', '250x188', thumb)
+        LOG.trace("Using convert executable at '%s' to generate thumbnails", convert_path)
+        THUMBNAIL_CMD = lambda full, thumb: (convert_path, full, '-thumbnail', '250x188', thumb)
 
 if THUMBNAIL_CMD is None and platform.system() == 'Darwin':
     # macOS only: fallback to sips if ImageMagick, e.g. from MacPorts or Homebrew is not found on macOS. sips is a system
     # executable that should be available on all macOS systems.
-    sips_path = spawn.find_executable('sips')
+    sips_path = shutil.which('sips')
     if sips_path:
-        LOG.trace('Using sips executable at \'%s\' to generate thumbnails'
-                  % sips_path)
-        THUMBNAIL_CMD = lambda full, thumb : (sips_path, '-Z', '250', '--out', thumb, full)
+        LOG.trace("Using sips executable at '%s' to generate thumbnails", sips_path)
+        THUMBNAIL_CMD = lambda full, thumb: (sips_path, '-Z', '250', '--out', thumb, full)
 
 if THUMBNAIL_CMD is None:
     LOG.warning('Could not find the ImageMagick \'convert\' or macOS \'sips\' command. '

pipeline/infrastructure/tablereader.py

@@ -1386,7 +1386,9 @@ def get_state_factory(msmd):
 
         epoch_start = me.epoch(time_ref, qa.quantity(scan_start, time_unit))
         str_start = qa.time(epoch_start['m0'], form=['fits'])[0]
-        dt_start = datetime.datetime.strptime(str_start, '%Y-%m-%dT%H:%M:%S')
+        # https://casadocs.readthedocs.io/en/latest/api/tt/casatools.quanta.html#casatools.quanta.quanta.time
+        # The 'fits' specification returns a string in ISO 8601 format with no subsecond precision
+        dt_start = datetime.datetime.fromisoformat(str_start).replace(tzinfo=datetime.timezone.utc)
 
         return domain.state.StateFactory(facility, dt_start)        
 

pipeline/infrastructure/utils/casa_data_test.py

@@ -1,18 +1,19 @@
-from datetime import datetime
 import re
+from datetime import datetime, timezone
+
 import pytest
+
 from .. import casa_tools
 from .casa_data import (
+    IERSInfo,
+    from_mjd_to_datetime,
     get_file_md5,
-    get_iso_mtime,
-    get_solar_system_model_files,
     get_filename_info,
+    get_iso_mtime,
     get_object_info_string,
-    IERSInfo,
-    from_mjd_to_datetime
+    get_solar_system_model_files,
 )
 
-
 # Validation of ISO 8601 strings
 REGEX_ISO8601 = (
     r'^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T'
@@ -144,9 +145,9 @@ def test_get_IERS_info():
 @skip_if_no_data_repo
 def test_validate_date_method():
     iers_info = IERSInfo(iers_path=GEODETIC_PATH)
-    assert iers_info.validate_date(datetime(2020, 12, 1, 0, 0))
-    assert not iers_info.validate_date(datetime(2021, 12, 1, 0, 0))
-    assert iers_info.validate_date(datetime(2019, 12, 1, 0, 0))
+    assert iers_info.validate_date(datetime(2020, 12, 1, 0, 0, tzinfo=timezone.utc))
+    assert not iers_info.validate_date(datetime(2021, 12, 1, 0, 0, tzinfo=timezone.utc))
+    assert iers_info.validate_date(datetime(2019, 12, 1, 0, 0, tzinfo=timezone.utc))
 
 
 @skip_if_no_data_repo
@@ -155,24 +156,25 @@ def test_string_representation_for_IERS_info():
     assert str(iers_info) == (
         'IERS table information => {"versions": {"IERSpredict": "0623.0351", '
         '"IERSeop2000": "0001.0144"}, "IERSeop2000_last_MJD": 59184.0, '
-        '"IERSeop2000_last": "2020-12-01 00:00:00", "IERSpredict_last": "2021-04-25 00:00:00"}'
+        '"IERSeop2000_last": "2020-12-01 00:00:00+00:00", "IERSpredict_last": "2021-04-25 00:00:00+00:00"}'
     )
 
+
 @skip_if_no_data_repo
 def test_date_message_type():
     iers_info = IERSInfo(iers_path=GEODETIC_PATH)
     # Before the end of IERSeop2000
-    assert iers_info.date_message_type(datetime(2019, 12, 1, 0, 0)) == "GOOD"
+    assert iers_info.date_message_type(datetime(2019, 12, 1, 0, 0, tzinfo=timezone.utc)) == "GOOD"
     # At the end of IERSeop2000
-    assert iers_info.date_message_type(datetime(2020, 12, 1, 0, 0)) == "GOOD"
+    assert iers_info.date_message_type(datetime(2020, 12, 1, 0, 0, tzinfo=timezone.utc)) == "GOOD"
     # Between the end of IERSeop2000 and IERSeop2000+3 months
-    assert iers_info.date_message_type(datetime(2020, 12, 2, 0, 0)) == "INFO"
+    assert iers_info.date_message_type(datetime(2020, 12, 2, 0, 0, tzinfo=timezone.utc)) == "INFO"
     # Between IERSeop2000 + 3 months and the end of IERSpredict
-    assert iers_info.date_message_type(datetime(2021, 3, 25, 0, 0)) == "WARN"
+    assert iers_info.date_message_type(datetime(2021, 3, 25, 0, 0, tzinfo=timezone.utc)) == "WARN"
     # At the end of IERSpredict
-    assert iers_info.date_message_type(datetime(2021, 4, 25, 0, 0)) == "WARN"
+    assert iers_info.date_message_type(datetime(2021, 4, 25, 0, 0, tzinfo=timezone.utc)) == "WARN"
     # After the end of IERSpredict
-    assert iers_info.date_message_type(datetime(2021, 12, 1, 0, 0)) == "CRITICAL"
+    assert iers_info.date_message_type(datetime(2021, 12, 1, 0, 0, tzinfo=timezone.utc)) == "CRITICAL"
 
 
 def test_date_message_type_when_data_is_not_found():
@@ -181,7 +183,7 @@ def test_date_message_type_when_data_is_not_found():
 
 
 def test_from_mjd_to_datetime():
-    assert from_mjd_to_datetime(59184.0) == datetime(2020, 12, 1, 0, 0)
+    assert from_mjd_to_datetime(59184.0) == datetime(2020, 12, 1, 0, 0, tzinfo=timezone.utc)
 
 
 def test_IERSInfo_when_data_is_not_found():
@@ -199,9 +201,9 @@ def test_get_IERS_info_when_data_is_not_found():
 
 def test_validate_date_method_when_data_is_not_found():
     iers_info = IERSInfo(iers_path=WRONG_GEODETIC_PATH)
-    assert not iers_info.validate_date(datetime(2020, 12, 1, 0, 0))
-    assert not iers_info.validate_date(datetime(2021, 12, 1, 0, 0))
-    assert not iers_info.validate_date(datetime(2019, 12, 1, 0, 0))
+    assert not iers_info.validate_date(datetime(2020, 12, 1, 0, 0, tzinfo=timezone.utc))
+    assert not iers_info.validate_date(datetime(2021, 12, 1, 0, 0, tzinfo=timezone.utc))
+    assert not iers_info.validate_date(datetime(2019, 12, 1, 0, 0, tzinfo=timezone.utc))
 
 
 def test_string_representation_for_IERS_info_when_data_is_not_found():

pipeline/infrastructure/utils/conversion.py

@@ -175,17 +175,6 @@ def flatten(l: Sequence[Any]) -> Iterator[Any]:
             yield el
 
 
-def unix_seconds_to_datetime(unix_secs: list[int | float]) -> list[datetime.datetime]:
-    """Convert list of UNIX epoch times to a list of equivalent datetime objects.
-
-    Args:
-        unix_secs: list of elapsed seconds since 1970-01-01.
-    Returns:
-        List of equivalent Python datetime objects.
-    """
-    return [datetime.datetime.utcfromtimestamp(s) for s in unix_secs]
-
-
 def mjd_seconds_to_datetime(mjd_secs: list[int | float]) -> list[datetime.datetime]:
     """Convert list of MJD seconds to a list of equivalent datetime objects.
 
@@ -197,10 +186,9 @@ def mjd_seconds_to_datetime(mjd_secs: list[int | float]) -> list[datetime.dateti
     Returns:
         List of equivalent Python datetime objects.
     """
-    # 1970-01-01 is JD 40587. 86400 = seconds in a day
-    unix_offset = 40587 * 86400
-    mjd_secs_with_offsets = [s - unix_offset for s in mjd_secs]
-    return unix_seconds_to_datetime(mjd_secs_with_offsets)
+    # MJD epoch is 1858-11-17 00:00:00 UTC
+    mjd_epoch = datetime.datetime(1858, 11, 17, tzinfo=datetime.timezone.utc)
+    return [mjd_epoch + datetime.timedelta(seconds=s) for s in mjd_secs]
 
 
 def get_epoch_as_datetime(epoch: dict) -> datetime.datetime:
@@ -225,7 +213,7 @@ def get_epoch_as_datetime(epoch: dict) -> datetime.datetime:
     t = mt.getvalue(epoch_utc)['m0']
     t = qt.sub(t, base_time)
     t = qt.convert(t, 's')
-    t = datetime.datetime.utcfromtimestamp(qt.getvalue(t)[0])
+    t = datetime.datetime.fromtimestamp(qt.getvalue(t)[0], datetime.timezone.utc)
 
     return t