diff --git a/apps/transport/lib/irve/coordinate_correction.ex b/apps/transport/lib/irve/coordinate_correction.ex index 5d04bd6588..17ba7b85da 100644 --- a/apps/transport/lib/irve/coordinate_correction.ex +++ b/apps/transport/lib/irve/coordinate_correction.ex @@ -17,7 +17,8 @@ defmodule Transport.IRVE.CoordinateCorrection do @doc """ Corrects inverted coordinates in a DataFrame with `longitude` and `latitude` float - columns. Adds `consolidated_is_lon_lat_correct` (`false` = row was swapped). + columns. Adds `warning_lon_lat_inverted` (`true` = row was swapped). The + `consolidated_is_lon_lat_correct` flag is derived later in the pipeline as its negation. iex> df = Explorer.DataFrame.new(longitude: [2.35, 48.85, 55.4], latitude: [48.85, 2.35, -21.1]) iex> r = Transport.IRVE.CoordinateCorrection.detect_and_correct(df) @@ -25,20 +26,29 @@ defmodule Transport.IRVE.CoordinateCorrection do [2.35, 2.35, 55.4] iex> Explorer.Series.to_list(r["latitude"]) [48.85, 48.85, -21.1] - iex> Explorer.Series.to_list(r["consolidated_is_lon_lat_correct"]) - [true, false, true] + iex> Explorer.Series.to_list(r["warning_lon_lat_inverted"]) + [false, true, false] + + Rows whose coordinates don't parse (`nil`) are left untouched and raise no warning: + + iex> df = Explorer.DataFrame.new(longitude: [2.35, nil], latitude: [48.85, nil]) + iex> r = Transport.IRVE.CoordinateCorrection.detect_and_correct(df) + iex> Explorer.Series.to_list(r["warning_lon_lat_inverted"]) + [false, false] """ def detect_and_correct(%DF{} = df) do DF.mutate_with(df, fn df -> lon = df["longitude"] lat = df["latitude"] - inverted = inverted?(lon, lat) + # `nil` coordinates yield a `nil` predicate, which would make `Series.select/3` panic; + # treat them as "not inverted" (no swap, no warning). + inverted = inverted?(lon, lat) |> Series.fill_missing(false) %{ longitude: Series.select(inverted, lat, lon), latitude: Series.select(inverted, lon, lat), - consolidated_is_lon_lat_correct: Series.not(inverted) + warning_lon_lat_inverted: inverted } end) end diff --git a/apps/transport/lib/irve/data_frame.ex b/apps/transport/lib/irve/data_frame.ex index face78983e..82b86ce31f 100644 --- a/apps/transport/lib/irve/data_frame.ex +++ b/apps/transport/lib/irve/data_frame.ex @@ -225,10 +225,12 @@ defmodule Transport.IRVE.DataFrame do https://schema.data.gouv.fr/etalab/schema-irve-statique/2.3.1/documentation.html#propriete-coordonneesxy The `preprocess_xy_coordinates` method attempts to remap that to 2 separate `x`, `y` fields, properly parsed. + The original `coordonneesXY` field is kept for reference, as it is used in the validation summary for warning samples. iex> Explorer.DataFrame.new([%{coordonneesXY: "[47.39,-0.80]"}]) |> Transport.IRVE.DataFrame.preprocess_xy_coordinates() #Explorer.DataFrame< - Polars[1 x 2] + Polars[1 x 3] + coordonneesXY string ["[47.39,-0.80]"] longitude f64 [47.39] latitude f64 [-0.8] > @@ -237,7 +239,8 @@ defmodule Transport.IRVE.DataFrame do iex> Explorer.DataFrame.new([%{coordonneesXY: "[43.958037, 4.764347]"}]) |> Transport.IRVE.DataFrame.preprocess_xy_coordinates() #Explorer.DataFrame< - Polars[1 x 2] + Polars[1 x 3] + coordonneesXY string ["[43.958037, 4.764347]"] longitude f64 [43.958037] latitude f64 [4.764347] > @@ -246,7 +249,8 @@ defmodule Transport.IRVE.DataFrame do iex> Explorer.DataFrame.new([%{coordonneesXY: " [6.128405 , 48.658737] "}]) |> Transport.IRVE.DataFrame.preprocess_xy_coordinates() #Explorer.DataFrame< - Polars[1 x 2] + Polars[1 x 3] + coordonneesXY string [" [6.128405 , 48.658737] "] longitude f64 [6.128405] latitude f64 [48.658737] > @@ -255,7 +259,8 @@ defmodule Transport.IRVE.DataFrame do iex> Explorer.DataFrame.new([%{coordonneesXY: "[43.306241,\t-0.332879]"}]) |> Transport.IRVE.DataFrame.preprocess_xy_coordinates() #Explorer.DataFrame< - Polars[1 x 2] + Polars[1 x 3] + coordonneesXY string ["[43.306241,\\t-0.332879]"] longitude f64 [43.306241] latitude f64 [-0.332879] > @@ -263,17 +268,18 @@ defmodule Transport.IRVE.DataFrame do The function is resilient to weird values: iex> Explorer.DataFrame.new([%{coordonneesXY: ""}, %{coordonneesXY: "[hello, 9]"}, %{coordonneesXY: "hello"}]) |> Transport.IRVE.DataFrame.preprocess_xy_coordinates() #Explorer.DataFrame< - Polars[3 x 2] + Polars[3 x 3] + coordonneesXY string ["", "[hello, 9]", "hello"] longitude f64 [nil, nil, nil] latitude f64 [nil, 9.0, nil] > """ def preprocess_xy_coordinates(df) do df - |> Explorer.DataFrame.mutate(coordonneesXY: coordonneesXY |> strip("[] ")) |> Explorer.DataFrame.mutate_with(fn df -> %{ - coords: Explorer.Series.split_into(df[:coordonneesXY], ",", [:longitude, :latitude]) + coords: + Explorer.Series.split_into(df[:coordonneesXY] |> Explorer.Series.strip("[] "), ",", [:longitude, :latitude]) } end) |> Explorer.DataFrame.unnest(:coords) @@ -285,7 +291,6 @@ defmodule Transport.IRVE.DataFrame do latitude: Explorer.Series.cast(df[:latitude], {:f, 64}) ] end) - |> Explorer.DataFrame.discard(:coordonneesXY) end # just what we've needed so far diff --git a/apps/transport/lib/irve/processing.ex b/apps/transport/lib/irve/processing.ex index e1f0794be3..becace00f6 100644 --- a/apps/transport/lib/irve/processing.ex +++ b/apps/transport/lib/irve/processing.ex @@ -4,6 +4,8 @@ defmodule Transport.IRVE.Processing do and preprocess (to some extent) a raw CSV binary body. """ + require Explorer.DataFrame + @doc """ Prepares the DataFrame without any type casting, all columns remain strings. This is useful for validation purposes. @@ -23,15 +25,17 @@ defmodule Transport.IRVE.Processing do Casts an uncasted (all-strings) validated DataFrame — the output of `read_as_uncasted_data_frame/1` enriched with validation columns — to a typed, insert-ready frame. + Coordinates are already parsed and corrected during validation (`longitude`/`latitude` + + `consolidated_is_lon_lat_correct`), so we only cast the remaining schema columns here. + Only fully-valid frames are expected here (whole-file gate): every value is castable by construction. A cast failure means the validator and this cast disagree, which is a bug — we let it raise. """ def cast_validated_frame(dataframe) do dataframe + |> Explorer.DataFrame.mutate(consolidated_is_lon_lat_correct: not warning_lon_lat_inverted) |> cast_to_schema_dtypes() - |> preprocess_coordinates() - |> Transport.IRVE.CoordinateCorrection.detect_and_correct() |> select_fields() end diff --git a/apps/transport/lib/irve/validator/data_frame_validation.ex b/apps/transport/lib/irve/validator/data_frame_validation.ex index dcde51970c..6eed855d70 100644 --- a/apps/transport/lib/irve/validator/data_frame_validation.ex +++ b/apps/transport/lib/irve/validator/data_frame_validation.ex @@ -83,22 +83,23 @@ defmodule Transport.IRVE.Validator.DataFrameValidation do end @doc """ - Adds custom warning columns. + Adds custom warning columns, and corrects the data in-place for the next steps of the pipeline. We only apply warnings to valid values to avoid noise. """ - def setup_computed_warning_columns(%Explorer.DataFrame{} = df) do + def setup_warning_columns_and_correct_data(%Explorer.DataFrame{} = df) do df - |> setup_inverted_lon_lat_warning_column() + |> inverted_lon_lat_setup_warning_and_correct_data() end - def setup_inverted_lon_lat_warning_column(%Explorer.DataFrame{} = df) do - warning = - Explorer.DataFrame.select(df, ["coordonneesXY"]) - # The next line is safe and shouldn’t raise errors even with weird values. - |> Transport.IRVE.Processing.preprocess_coordinates() - |> Transport.IRVE.CoordinateCorrection.inverted?() - |> Explorer.Series.and(df["check_column_coordonneesXY_valid"]) - - Explorer.DataFrame.put(df, "warning_lon_lat_inverted", warning) + @doc """ + Parses and corrects the coordinates once, here in the validation pipeline: adds + `longitude`/`latitude` (swapped where inverted) and `warning_lon_lat_inverted`. + If the coordinates can’t be parsed, there is no warning. + """ + def inverted_lon_lat_setup_warning_and_correct_data(%Explorer.DataFrame{} = df) do + df + # The next line is safe and shouldn’t raise errors even with weird values. + |> Transport.IRVE.Processing.preprocess_coordinates() + |> Transport.IRVE.CoordinateCorrection.detect_and_correct() end end diff --git a/apps/transport/lib/irve/validator/validator.ex b/apps/transport/lib/irve/validator/validator.ex index 281b96cbe5..a1eed0cea7 100644 --- a/apps/transport/lib/irve/validator/validator.ex +++ b/apps/transport/lib/irve/validator/validator.ex @@ -1,6 +1,13 @@ defmodule Transport.IRVE.Validator do @moduledoc """ Central entry point for IRVE file validation (currently working on `DataFrame`). + Use main function `validate_and_summarize/1` to get a summary of the validation and the validated DataFrame. + The validated DataFrame is not casted to the schema dtypes, but contains: + - the original columns, as strings (uncasted) + - additional columns for each field, indicating if the value is valid or not + - a `check_row_valid` column indicating if the row is valid or not + - additional columns for warnings, indicating if the value is valid but has a warning + - corrected values for some fields (e.g. inverted coordinates, missing optional columns, etc.) """ @unexpected_error_prefix "Unexpected error: " @@ -11,7 +18,7 @@ defmodule Transport.IRVE.Validator do df |> Transport.IRVE.Validator.DataFrameValidation.setup_computed_field_validation_columns(schema) |> Transport.IRVE.Validator.DataFrameValidation.setup_computed_row_validation_column() - |> Transport.IRVE.Validator.DataFrameValidation.setup_computed_warning_columns() + |> Transport.IRVE.Validator.DataFrameValidation.setup_warning_columns_and_correct_data() end @doc """ diff --git a/apps/transport/test/transport/irve/validation/data_frame_validation_test.exs b/apps/transport/test/transport/irve/validation/data_frame_validation_test.exs index 686587b4d8..3877b196df 100644 --- a/apps/transport/test/transport/irve/validation/data_frame_validation_test.exs +++ b/apps/transport/test/transport/irve/validation/data_frame_validation_test.exs @@ -284,6 +284,13 @@ defmodule Transport.IRVE.Validator.DataFrameValidation.WarningsTest do assert Explorer.Series.to_list(result["warning_lon_lat_inverted"]) == [false] end + test "corrects coordinates and exposes a warning" do + result = run_with_xy("[48.85, 2.35]") + assert Explorer.Series.to_list(result["longitude"]) == [2.35] + assert Explorer.Series.to_list(result["latitude"]) == [48.85] + assert Explorer.Series.to_list(result["warning_lon_lat_inverted"]) == [true] + end + defp run_with_xy(xy) do [DB.Factory.IRVE.generate_row(%{"coordonneesXY" => xy}) |> Parent.stringify_row()] |> Explorer.DataFrame.new()