[DO NOT MERGE] Consolidation IRVE : une seule passe de correction des coordonnées

apps/transport/lib/irve/coordinate_correction.ex

@@ -17,28 +17,38 @@ defmodule Transport.IRVE.CoordinateCorrection do
 
   @doc """
   Corrects inverted coordinates in a DataFrame with `longitude` and `latitude` float
-  columns. Adds `consolidated_is_lon_lat_correct` (`false` = row was swapped).
+  columns. Adds `warning_lon_lat_inverted` (`true` = row was swapped). The
+  `consolidated_is_lon_lat_correct` flag is derived later in the pipeline as its negation.
 
       iex> df = Explorer.DataFrame.new(longitude: [2.35, 48.85, 55.4], latitude: [48.85, 2.35, -21.1])
       iex> r = Transport.IRVE.CoordinateCorrection.detect_and_correct(df)
       iex> Explorer.Series.to_list(r["longitude"])
       [2.35, 2.35, 55.4]
       iex> Explorer.Series.to_list(r["latitude"])
       [48.85, 48.85, -21.1]
-      iex> Explorer.Series.to_list(r["consolidated_is_lon_lat_correct"])
-      [true, false, true]
+      iex> Explorer.Series.to_list(r["warning_lon_lat_inverted"])
+      [false, true, false]
+
+  Rows whose coordinates don't parse (`nil`) are left untouched and raise no warning:
+
+      iex> df = Explorer.DataFrame.new(longitude: [2.35, nil], latitude: [48.85, nil])
+      iex> r = Transport.IRVE.CoordinateCorrection.detect_and_correct(df)
+      iex> Explorer.Series.to_list(r["warning_lon_lat_inverted"])
+      [false, false]
 
   """
   def detect_and_correct(%DF{} = df) do
     DF.mutate_with(df, fn df ->
       lon = df["longitude"]
       lat = df["latitude"]
-      inverted = inverted?(lon, lat)
+      # `nil` coordinates yield a `nil` predicate, which would make `Series.select/3` panic;
+      # treat them as "not inverted" (no swap, no warning).
+      inverted = inverted?(lon, lat) |> Series.fill_missing(false)
 
       %{
         longitude: Series.select(inverted, lat, lon),
         latitude: Series.select(inverted, lon, lat),
-        consolidated_is_lon_lat_correct: Series.not(inverted)
+        warning_lon_lat_inverted: inverted
       }
     end)
   end

apps/transport/lib/irve/data_frame.ex

@@ -225,10 +225,12 @@ defmodule Transport.IRVE.DataFrame do
   https://schema.data.gouv.fr/etalab/schema-irve-statique/2.3.1/documentation.html#propriete-coordonneesxy
 
   The `preprocess_xy_coordinates` method attempts to remap that to 2 separate `x`, `y` fields, properly parsed.
+  The original `coordonneesXY` field is kept for reference, as it is used in the validation summary for warning samples.
 
   iex> Explorer.DataFrame.new([%{coordonneesXY: "[47.39,-0.80]"}]) |> Transport.IRVE.DataFrame.preprocess_xy_coordinates()
   #Explorer.DataFrame<
-    Polars[1 x 2]
+    Polars[1 x 3]
+    coordonneesXY string ["[47.39,-0.80]"]
     longitude f64 [47.39]
     latitude f64 [-0.8]
   >
@@ -237,7 +239,8 @@ defmodule Transport.IRVE.DataFrame do
 
   iex> Explorer.DataFrame.new([%{coordonneesXY: "[43.958037, 4.764347]"}]) |> Transport.IRVE.DataFrame.preprocess_xy_coordinates()
   #Explorer.DataFrame<
-    Polars[1 x 2]
+    Polars[1 x 3]
+    coordonneesXY string ["[43.958037, 4.764347]"]
     longitude f64 [43.958037]
     latitude f64 [4.764347]
   >
@@ -246,7 +249,8 @@ defmodule Transport.IRVE.DataFrame do
 
   iex> Explorer.DataFrame.new([%{coordonneesXY: " [6.128405 , 48.658737] "}]) |> Transport.IRVE.DataFrame.preprocess_xy_coordinates()
   #Explorer.DataFrame<
-    Polars[1 x 2]
+    Polars[1 x 3]
+    coordonneesXY string [" [6.128405 , 48.658737] "]
     longitude f64 [6.128405]
     latitude f64 [48.658737]
   >
@@ -255,25 +259,27 @@ defmodule Transport.IRVE.DataFrame do
 
     iex> Explorer.DataFrame.new([%{coordonneesXY: "[43.306241,\t-0.332879]"}]) |> Transport.IRVE.DataFrame.preprocess_xy_coordinates()
     #Explorer.DataFrame<
-      Polars[1 x 2]
+      Polars[1 x 3]
+      coordonneesXY string ["[43.306241,\\t-0.332879]"]
       longitude f64 [43.306241]
       latitude f64 [-0.332879]
     >
 
   The function is resilient to weird values:
   iex> Explorer.DataFrame.new([%{coordonneesXY: ""}, %{coordonneesXY: "[hello, 9]"}, %{coordonneesXY: "hello"}]) |> Transport.IRVE.DataFrame.preprocess_xy_coordinates()
   #Explorer.DataFrame<
-    Polars[3 x 2]
+    Polars[3 x 3]
+    coordonneesXY string ["", "[hello, 9]", "hello"]
     longitude f64 [nil, nil, nil]
     latitude f64 [nil, 9.0, nil]
   >
   """
   def preprocess_xy_coordinates(df) do
     df
-    |> Explorer.DataFrame.mutate(coordonneesXY: coordonneesXY |> strip("[] "))
     |> Explorer.DataFrame.mutate_with(fn df ->
       %{
-        coords: Explorer.Series.split_into(df[:coordonneesXY], ",", [:longitude, :latitude])
+        coords:
+          Explorer.Series.split_into(df[:coordonneesXY] |> Explorer.Series.strip("[] "), ",", [:longitude, :latitude])
       }
     end)
     |> Explorer.DataFrame.unnest(:coords)
@@ -285,7 +291,6 @@ defmodule Transport.IRVE.DataFrame do
         latitude: Explorer.Series.cast(df[:latitude], {:f, 64})
       ]
     end)
-    |> Explorer.DataFrame.discard(:coordonneesXY)
   end
 
   # just what we've needed so far

apps/transport/lib/irve/processing.ex

@@ -4,6 +4,8 @@ defmodule Transport.IRVE.Processing do
   and preprocess (to some extent) a raw CSV binary body.
   """
 
+  require Explorer.DataFrame
+
   @doc """
   Prepares the DataFrame without any type casting, all columns remain strings.
   This is useful for validation purposes.
@@ -23,15 +25,17 @@ defmodule Transport.IRVE.Processing do
   Casts an uncasted (all-strings) validated DataFrame — the output of
   `read_as_uncasted_data_frame/1` enriched with validation columns — to a typed, insert-ready frame.
 
+  Coordinates are already parsed and corrected during validation (`longitude`/`latitude` +
+  `consolidated_is_lon_lat_correct`), so we only cast the remaining schema columns here.
+
   Only fully-valid frames are expected here (whole-file gate): every value is castable by
   construction. A cast failure means the validator and this cast disagree, which is a bug —
   we let it raise.
   """
   def cast_validated_frame(dataframe) do
     dataframe
+    |> Explorer.DataFrame.mutate(consolidated_is_lon_lat_correct: not warning_lon_lat_inverted)
     |> cast_to_schema_dtypes()
-    |> preprocess_coordinates()
-    |> Transport.IRVE.CoordinateCorrection.detect_and_correct()
     |> select_fields()
   end
 

apps/transport/lib/irve/validator/data_frame_validation.ex

@@ -83,22 +83,23 @@ defmodule Transport.IRVE.Validator.DataFrameValidation do
   end
 
   @doc """
-  Adds custom warning columns.
+  Adds custom warning columns, and corrects the data in-place for the next steps of the pipeline.
   We only apply warnings to valid values to avoid noise.
   """
-  def setup_computed_warning_columns(%Explorer.DataFrame{} = df) do
+  def setup_warning_columns_and_correct_data(%Explorer.DataFrame{} = df) do
     df
-    |> setup_inverted_lon_lat_warning_column()
+    |> inverted_lon_lat_setup_warning_and_correct_data()
   end
 
-  def setup_inverted_lon_lat_warning_column(%Explorer.DataFrame{} = df) do
-    warning =
-      Explorer.DataFrame.select(df, ["coordonneesXY"])
-      # The next line is safe and shouldn’t raise errors even with weird values.
-      |> Transport.IRVE.Processing.preprocess_coordinates()
-      |> Transport.IRVE.CoordinateCorrection.inverted?()
-      |> Explorer.Series.and(df["check_column_coordonneesXY_valid"])
-
-    Explorer.DataFrame.put(df, "warning_lon_lat_inverted", warning)
+  @doc """
+  Parses and corrects the coordinates once, here in the validation pipeline: adds
+  `longitude`/`latitude` (swapped where inverted) and `warning_lon_lat_inverted`.
+  If the coordinates can’t be parsed, there is no warning.
+  """
+  def inverted_lon_lat_setup_warning_and_correct_data(%Explorer.DataFrame{} = df) do
+    df
+    # The next line is safe and shouldn’t raise errors even with weird values.
+    |> Transport.IRVE.Processing.preprocess_coordinates()
+    |> Transport.IRVE.CoordinateCorrection.detect_and_correct()
   end
 end

apps/transport/lib/irve/validator/validator.ex

@@ -1,6 +1,13 @@
 defmodule Transport.IRVE.Validator do
   @moduledoc """
   Central entry point for IRVE file validation (currently working on `DataFrame`).
+  Use main function `validate_and_summarize/1` to get a summary of the validation and the validated DataFrame.
+  The validated DataFrame is not casted to the schema dtypes, but contains:
+  - the original columns, as strings (uncasted)
+  - additional columns for each field, indicating if the value is valid or not
+  - a `check_row_valid` column indicating if the row is valid or not
+  - additional columns for warnings, indicating if the value is valid but has a warning
+  - corrected values for some fields (e.g. inverted coordinates, missing optional columns, etc.)
   """
 
   @unexpected_error_prefix "Unexpected error: "
@@ -11,7 +18,7 @@ defmodule Transport.IRVE.Validator do
     df
     |> Transport.IRVE.Validator.DataFrameValidation.setup_computed_field_validation_columns(schema)
     |> Transport.IRVE.Validator.DataFrameValidation.setup_computed_row_validation_column()
-    |> Transport.IRVE.Validator.DataFrameValidation.setup_computed_warning_columns()
+    |> Transport.IRVE.Validator.DataFrameValidation.setup_warning_columns_and_correct_data()
   end
 
   @doc """

apps/transport/test/transport/irve/validation/data_frame_validation_test.exs

@@ -284,6 +284,13 @@ defmodule Transport.IRVE.Validator.DataFrameValidation.WarningsTest do
     assert Explorer.Series.to_list(result["warning_lon_lat_inverted"]) == [false]
   end
 
+  test "corrects coordinates and exposes a warning" do
+    result = run_with_xy("[48.85, 2.35]")
+    assert Explorer.Series.to_list(result["longitude"]) == [2.35]
+    assert Explorer.Series.to_list(result["latitude"]) == [48.85]
+    assert Explorer.Series.to_list(result["warning_lon_lat_inverted"]) == [true]
+  end
+
   defp run_with_xy(xy) do
     [DB.Factory.IRVE.generate_row(%{"coordonneesXY" => xy}) |> Parent.stringify_row()]
     |> Explorer.DataFrame.new()