Merge pull request #396 from catalyst-cooperative/string-datetimes

Treat datetime columns as strings

Author: zaneselvans

.github/release.yml

@@ -0,0 +1,10 @@
+---
+changelog:
+  exclude:
+    authors:
+      - catalyst-workflow-triggerer
+      - dependabot
+      - pre-commit-ci
+      - pudlbot
+    labels:
+      - dependencies

.pre-commit-config.yaml

@@ -28,7 +28,7 @@ repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.14.7
     hooks:
-      - id: ruff
+      - id: ruff-check
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
 

src/ferc_xbrl_extractor/datapackage.py

@@ -9,7 +9,7 @@
 import stringcase
 from pydantic import BaseModel
 
-from ferc_xbrl_extractor.helpers import get_logger, parse_dates
+from ferc_xbrl_extractor.helpers import get_logger
 from ferc_xbrl_extractor.instance import Instance
 from ferc_xbrl_extractor.taxonomy import Concept, LinkRole, Taxonomy
 
@@ -118,7 +118,7 @@ def __hash__(self):
     "number": "Float64",
     "integer": "Int64",
     "boolean": "boolean",
-    "date": "datetime64[ms]",
+    "date": "string",
     "duration": "string",
     "year": "Int64",
 }
@@ -133,7 +133,7 @@ def __hash__(self):
     "number": float,
     "boolean": bool,
     "duration": str,
-    "date": parse_dates,
+    "date": str,
 }
 """
 Map callables to schema field type to convert parsed values (Data Package `field.type`).

src/ferc_xbrl_extractor/helpers.py

@@ -2,20 +2,6 @@
 
 import logging
 
-import pandas as pd
-
-
-def parse_dates(date_str: str) -> pd.Timestamp:
-    """Helper to normalize date strings/parse in a consistent way."""
-    try:
-        if "24:00:00" in date_str:
-            return pd.to_datetime(
-                date_str.replace("24:00:00", "00:00:00")
-            ) + pd.Timedelta(days=1)
-        return pd.to_datetime(date_str)
-    except ValueError:
-        return pd.NaT
-
 
 def get_logger(name: str) -> logging.Logger:
     """Helper function to append 'catalystcoop' to logger name and return logger."""

src/ferc_xbrl_extractor/instance.py

@@ -16,7 +16,7 @@
 from lxml.etree import _Element as Element  # nosec: B410
 from pydantic import BaseModel, field_validator
 
-from ferc_xbrl_extractor.helpers import get_logger, parse_dates
+from ferc_xbrl_extractor.helpers import get_logger
 
 XBRL_INSTANCE = "http://www.xbrl.org/2003/instance"
 XBRL_LINK = "http://www.xbrl.org/2003/linkbase"
@@ -179,12 +179,12 @@ def as_primary_key(self, filing_name: str, axes: list[str]) -> dict[str, str]:
 
         # Get date based on period type
         if self.period.instant:
-            date_dict = {"date": parse_dates(self.period.end_date)}
+            date_dict = {"date": self.period.end_date}
         else:
             date_dict = {
                 # Ignore type because start_date will always be str if duration period
-                "start_date": parse_dates(self.period.start_date),
-                "end_date": parse_dates(self.period.end_date),
+                "start_date": self.period.start_date,
+                "end_date": self.period.end_date,
             }
 
         return {

tests/integration/datapackage_test.py

@@ -97,7 +97,7 @@ def _create_schema(instant=True, axes=None):
                     'cid_4,EID1,filing,2023-01-01T00:00:01,2020-01-01,2020-12-31,"value 3","value 4",\n'
                 ),
                 dtype="string",
-                parse_dates=["publication_time", "start_date", "end_date"],
+                parse_dates=["publication_time"],
             ),
         ),
         (
@@ -111,7 +111,7 @@ def _create_schema(instant=True, axes=None):
                     'cid_5,EID1,filing,2023-01-01T00:00:01,2020-01-01,2020-12-31,"Dim 1 Value","value 9","value 10",\n'
                 ),
                 dtype="string",
-                parse_dates=["publication_time", "start_date", "end_date"],
+                parse_dates=["publication_time"],
             ),
         ),
         (
@@ -124,7 +124,7 @@ def _create_schema(instant=True, axes=None):
                     'cid_3,EID1,filing,2023-01-01T00:00:01,2021-12-31,"Dim 1 Value","ferc:Dimension2Value","value 7","value 8",\n'
                 ),
                 dtype="string",
-                parse_dates=["publication_time", "date"],
+                parse_dates=["publication_time"],
             ),
         ),
     ],
@@ -148,12 +148,5 @@ def test_construct_dataframe(table_schema, period, df, in_memory_filing):
         .drop("c_id", axis="columns")
         .reset_index()
     )
-    # Make sure date types use correct units
-    expected_df = expected_df.astype(
-        {
-            col: "datetime64[ms]"
-            for col in ["publication_time", "start_date", "end_date", "date"]
-            if col in expected_df.columns
-        }
-    )
+    expected_df = expected_df.astype({"publication_time": "string"})
     pd.testing.assert_frame_equal(expected_df, constructed_df)

tests/unit/instance_test.py

@@ -4,7 +4,6 @@
 import logging
 from collections import Counter
 
-import pandas as pd
 import pytest
 
 from ferc_xbrl_extractor.instance import (
@@ -80,18 +79,16 @@ def test_context_ids(test_context):
 
     assert context_ids.get("entity_id") == test_context.entity.identifier
     assert context_ids.get("filing_name") == "filing_name"
-    assert context_ids.get("date") == pd.to_datetime(test_context.period.end_date)
+    assert context_ids.get("date") == test_context.period.end_date
 
     # Change context to have a duration period, then change
     test_context.period.instant = False
     test_context.period.start_date = "2019-01-01"
 
     context_ids = test_context.as_primary_key("filing_name", axes)
 
-    assert context_ids.get("start_date") == pd.to_datetime(
-        test_context.period.start_date
-    )
-    assert context_ids.get("end_date") == pd.to_datetime(test_context.period.end_date)
+    assert context_ids.get("start_date") == test_context.period.start_date
+    assert context_ids.get("end_date") == test_context.period.end_date
 
 
 @pytest.mark.parametrize(