Résumé de validation IRVE (#5429)

Author: vdegove

apps/transport/lib/irve/validator/summary.ex

@@ -0,0 +1,25 @@
+defmodule Transport.IRVE.Validator.Summary do
+  @moduledoc """
+  Struct representing the human-actionable summary produced after IRVE validation.
+  """
+
+  @enforce_keys [
+    :valid,
+    :valid_row_count,
+    :invalid_row_count,
+    :total_row_count,
+    :file_level_errors,
+    :column_errors,
+    :error_samples
+  ]
+
+  defstruct [
+    :valid,
+    :valid_row_count,
+    :invalid_row_count,
+    :total_row_count,
+    :file_level_errors,
+    :column_errors,
+    :error_samples
+  ]
+end

apps/transport/lib/irve/validator/validator.ex

@@ -18,7 +18,6 @@ defmodule Transport.IRVE.Validator do
   so files that are not strictly valid may be considered as valid without any notice by this function.
   If you want to call a strict validator (no preprocessing), use `compute_validation/1` instead.
   """
-
   def validate(path, extension \\ ".csv") do
     # NOTE: for now, load the body in memory, because refactoring to get full streaming
     # is too involved for the current sprint deadline.
@@ -40,4 +39,92 @@ defmodule Transport.IRVE.Validator do
     df["check_row_valid"]
     |> Explorer.Series.all?()
   end
+
+  @doc """
+  Produces a human-actionable summary from the validated DataFrame returned by `compute_validation/1`.
+  Still an Elixir map, but the content should allow someone to know how to correct an invalid file.
+
+  Returns a map with:
+  - `valid` - whether the file is fully valid
+  - `valid_row_count` - number of valid rows
+  - `invalid_row_count` - number of invalid rows
+  - `column_errors` - map of field name => number of invalid rows for that column (only columns with at least one error)
+  - `error_samples` - up to 5 sample errors per errored column, each with `id_pdc_itinerance`, `column`, and `value`
+  """
+  def summarize(%Explorer.DataFrame{} = df) do
+    {valid_count, invalid_count} = summarize_total_counts(df)
+    column_errors = summarize_column_errors(df)
+    error_samples = error_samples(df, column_errors)
+
+    %Transport.IRVE.Validator.Summary{
+      valid: invalid_count == 0,
+      valid_row_count: valid_count,
+      invalid_row_count: invalid_count,
+      total_row_count: valid_count + invalid_count,
+      file_level_errors: [],
+      column_errors: column_errors,
+      error_samples: error_samples
+    }
+  end
+
+  @doc """
+  Combines `validate/2` and `summarize/1` into a single call, catching any file-level error
+  (bad encoding, wrong format, missing columns, etc.) instead of raising.
+
+  Returns the same map as `summarize/1` with an additional `file_level_error` key:
+  - `nil` when validation ran successfully (the file could still be invalid row-by-row)
+  - an error message string when a hard file-level error was caught
+
+  On a file-level error, `valid_row_count`, `invalid_row_count`, `column_errors`, and
+  `error_samples` are all `nil`/empty since there is no DataFrame to summarize from.
+  """
+  def validate_and_summarize(path, extension \\ ".csv") do
+    path
+    |> validate(extension)
+    |> summarize()
+  rescue
+    error ->
+      %Transport.IRVE.Validator.Summary{
+        valid: false,
+        valid_row_count: nil,
+        invalid_row_count: nil,
+        total_row_count: nil,
+        file_level_errors: [Exception.message(error)],
+        column_errors: %{},
+        error_samples: []
+      }
+  end
+
+  defp summarize_total_counts(df) do
+    valid_count = df["check_row_valid"] |> Explorer.Series.sum()
+    invalid_count = Explorer.DataFrame.n_rows(df) - valid_count
+    {valid_count, invalid_count}
+  end
+
+  defp summarize_column_errors(df) do
+    Transport.IRVE.StaticIRVESchema.field_names_list()
+    |> Enum.map(fn field_name -> {field_name, "check_column_#{field_name}_valid"} end)
+    |> Enum.flat_map(fn {field_name, check_col} ->
+      error_count = df[check_col] |> Explorer.Series.not() |> Explorer.Series.sum()
+      if error_count > 0, do: [{field_name, error_count}], else: []
+    end)
+    |> Map.new()
+  end
+
+  defp error_samples(df, column_errors) do
+    column_errors
+    |> Enum.flat_map(fn {field_name, _error_count} ->
+      check_col = "check_column_#{field_name}_valid"
+
+      df
+      |> Explorer.DataFrame.filter_with(&(&1[check_col] |> Explorer.Series.not()))
+      |> Explorer.DataFrame.select(["id_pdc_itinerance", field_name])
+      # Limit to 5 samples per error column
+      |> Explorer.DataFrame.head(5)
+      |> Explorer.DataFrame.to_rows()
+      |> Enum.map(fn row ->
+        %{id_pdc_itinerance: row["id_pdc_itinerance"], column: field_name, value: row[field_name]}
+      end)
+    end)
+  end
 end

apps/transport/test/transport/irve/validation/validator_test.exs

@@ -138,4 +138,76 @@ defmodule Transport.IRVE.ValidatorTest do
       assert Transport.IRVE.Validator.full_file_valid?(result)
     end)
   end
+
+  test "summarize/1 returns a valid summary for a valid file" do
+    csv_content = [DB.Factory.IRVE.generate_row()] |> DB.Factory.IRVE.to_csv_body()
+
+    with_tmp_file(csv_content, fn path ->
+      summary = Transport.IRVE.Validator.validate(path) |> Transport.IRVE.Validator.summarize()
+
+      assert %Transport.IRVE.Validator.Summary{
+               valid: true,
+               valid_row_count: 1,
+               invalid_row_count: 0,
+               total_row_count: 1,
+               file_level_errors: [],
+               column_errors: %{},
+               error_samples: []
+             } == summary
+    end)
+  end
+
+  test "summarize/1 reports column errors and caps error samples to 5 per column" do
+    # 10 rows with invalid puissance_nominale, 1 row with invalid nbre_pdc (valid puissance_nominale)
+    invalid_puissance_rows = for _ <- 1..10, do: DB.Factory.IRVE.generate_row(%{"puissance_nominale" => "not-a-number"})
+    invalid_nbre_pdc_row = DB.Factory.IRVE.generate_row(%{"nbre_pdc" => "not-a-number"})
+    csv_content = (invalid_puissance_rows ++ [invalid_nbre_pdc_row]) |> DB.Factory.IRVE.to_csv_body()
+
+    with_tmp_file(csv_content, fn path ->
+      summary = Transport.IRVE.Validator.validate(path) |> Transport.IRVE.Validator.summarize()
+
+      assert summary.valid == false
+      assert summary.valid_row_count == 0
+      assert summary.invalid_row_count == 11
+      assert summary.column_errors == %{"puissance_nominale" => 10, "nbre_pdc" => 1}
+
+      # 5 samples from puissance_nominale (capped) + 1 from nbre_pdc
+      assert length(summary.error_samples) == 6
+
+      assert summary.error_samples
+             |> Enum.filter(&(&1.column == "puissance_nominale"))
+             |> Enum.all?(&(&1.value == "not-a-number"))
+
+      assert [%{column: "nbre_pdc", value: "not-a-number", id_pdc_itinerance: _}] =
+               Enum.filter(summary.error_samples, &(&1.column == "nbre_pdc"))
+    end)
+  end
+
+  test "validate_and_summarize/1 returns a normal summary with file_level_error: nil for a valid file" do
+    csv_content = [DB.Factory.IRVE.generate_row()] |> DB.Factory.IRVE.to_csv_body()
+
+    with_tmp_file(csv_content, fn path ->
+      summary = Transport.IRVE.Validator.validate_and_summarize(path)
+
+      assert %{valid: true, valid_row_count: 1, invalid_row_count: 0, file_level_errors: []} = summary
+    end)
+  end
+
+  test "validate_and_summarize/1 returns an error summary instead of raising on a file-level error" do
+    with_tmp_file("PK\x03\x04" <> "some content", fn path ->
+      summary = Transport.IRVE.Validator.validate_and_summarize(path)
+
+      assert %{
+               valid: false,
+               valid_row_count: nil,
+               invalid_row_count: nil,
+               total_row_count: nil,
+               file_level_errors: [error_message],
+               column_errors: %{},
+               error_samples: []
+             } = summary
+
+      assert error_message =~ "zip"
+    end)
+  end
 end