Make _observations.py use polars

Author: akustn

src/ert/config/_observations.py

@@ -18,7 +18,6 @@
 )
 
 import numpy as np
-import pandas as pd
 import polars as pl
 import scipy as sp
 from pydantic import BaseModel, ConfigDict, Field, model_serializer
@@ -686,11 +685,7 @@ def from_csv(
                 f"The CSV file ({filename}) does not exist or is not accessible.",
                 filename,
             )
-        csv_file = pd.read_csv(
-            filename,
-            encoding="utf-8",
-            on_bad_lines="error",
-        )
+        csv_file = pl.read_csv(filename, encoding="utf-8")
 
         required_columns = {
             "WELL_NAME",
@@ -701,7 +696,7 @@ def from_csv(
             "EAST",
             "TVD",
         }
-        missing_required_columns = required_columns - set(csv_file.keys())
+        missing_required_columns = required_columns - set(csv_file.columns)
         if missing_required_columns:
             raise ObservationConfigError.with_context(
                 f"The rft observations file {filename} is missing required column(s) "
@@ -711,9 +706,10 @@ def from_csv(
 
         rft_observations = []
         invalid_observations = []
-        for row in csv_file.itertuples(index=True):
-            east_val = validate_float(str(row.EAST), "EAST")
-            north_val = validate_float(str(row.NORTH), "NORTH")
+        columns = set(csv_file.columns)
+        for index, row in enumerate(csv_file.iter_rows(named=True)):
+            east_val = validate_float(str(row["EAST"]), "EAST")
+            north_val = validate_float(str(row["NORTH"]), "NORTH")
             radius = radius if radius is not None else DEFAULT_LOCALIZATION_RADIUS
 
             shape_id = shape_registry.register(
@@ -725,22 +721,20 @@ def from_csv(
             )
 
             rft_observation = cls(
-                name=f"{observation_dict['name']}[{row.Index}]",
-                well=str(row.WELL_NAME),
-                date=str(row.DATE),
+                name=f"{observation_dict['name']}[{index}]",
+                well=str(row["WELL_NAME"]),
+                date=str(row["DATE"]),
                 property=observed_property,
-                value=validate_float(
-                    str(getattr(row, observed_property)), observed_property
-                ),
-                error=validate_float(str(row.ERROR), "ERROR"),
+                value=validate_float(str(row[observed_property]), observed_property),
+                error=validate_float(str(row["ERROR"]), "ERROR"),
                 east=east_val,
                 north=north_val,
                 shape_id=shape_id,
-                tvd=validate_float(str(row.TVD), "TVD"),
-                md=validate_float(str(row.MD), "MD") if "MD" in csv_file else None,
+                tvd=validate_float(str(row["TVD"]), "TVD"),
+                md=validate_float(str(row["MD"]), "MD") if "MD" in columns else None,
                 zone=(
-                    str(row.ZONE)
-                    if "ZONE" in csv_file and row.ZONE is not None
+                    str(row["ZONE"])
+                    if "ZONE" in columns and row["ZONE"] is not None
                     else None
                 ),
             )
@@ -1007,11 +1001,10 @@ class SeismicObservation(BaseObservation):
     TOLERANCE: ClassVar[float] = 0.1
 
     @staticmethod
-    def _load_observations(filepath: Path) -> pd.DataFrame:
-        df = pd.read_csv(
+    def _load_observations(filepath: Path) -> pl.DataFrame:
+        df = pl.read_csv(
             filepath,
             encoding="utf-8",
-            on_bad_lines="error",
         )
 
         required_columns = {
@@ -1020,7 +1013,7 @@ def _load_observations(filepath: Path) -> pd.DataFrame:
             "OBS",
             "OBS_ERROR",
         }
-        missing_required_columns = required_columns - set(df.keys())
+        missing_required_columns = required_columns - set(df.columns)
         if missing_required_columns:
             raise ObservationConfigError.with_context(
                 f"The seismic observations file {filepath} "
@@ -1119,11 +1112,11 @@ def from_obs_dict(
             boundary_id = shape_registry.register(boundary)
 
         seismic_observations = []
-        for row in df.itertuples():
-            east = validate_float(str(row.X_UTME), "X_UTME")
-            north = validate_float(str(row.Y_UTMN), "Y_UTMN")
-            value = validate_float(str(row.OBS), "OBS")
-            error = validate_float(str(row.OBS_ERROR), "OBS_ERROR")
+        for row in df.iter_rows(named=True):
+            east = validate_float(str(row["X_UTME"]), "X_UTME")
+            north = validate_float(str(row["Y_UTMN"]), "Y_UTMN")
+            value = validate_float(str(row["OBS"]), "OBS")
+            error = validate_float(str(row["OBS_ERROR"]), "OBS_ERROR")
 
             # Currently supports only default localization radius as behavior of
             # LOCALIZATION keyword is undefined. All shapes are being registered

tests/ert/unit_tests/config/test_observation_declaration.py

@@ -841,9 +841,9 @@ def test_that_seismic_observation_instantiates(file_context_token):
         create_seismic_observation(
             name="NAME",
             filepath=Path("obs.csv"),
-            east=461231.5537527473,
+            east=461231.55375274725,
             north=5933187.729869121,
-            value=-0.0003566695393886,
+            value=-0.00035666953938864876,
             error=0.005,
             shape_id=0,
             boundary_id=None,
@@ -853,7 +853,7 @@ def test_that_seismic_observation_instantiates(file_context_token):
             filepath=Path("obs.csv"),
             east=461156.9532936567,
             north=5933317.28138355,
-            value=-0.0005293887515127,
+            value=-0.0005293887515127136,
             error=0.005,
             shape_id=1,
             boundary_id=None,