Skip to content

Commit ef2f916

Browse files
vdegovethbar
vdegove
and
thbar
authored
IRVE : affichage d’avertissements concernant les inversions de coordonnées dans le validateur à la demande (#5541)
Co-authored-by: Thibaut Barrère <thibaut.barrere@gmail.com>
1 parent f6b56d7 commit ef2f916

13 files changed

Lines changed: 300 additions & 16 deletions

File tree

‎apps/transport/lib/irve/coordinate_correction.ex‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ defmodule Transport.IRVE.CoordinateCorrection do
6262
[false, true, false, true]
6363
6464
"""
65+
def inverted?(%DF{} = df), do: inverted?(df["longitude"], df["latitude"])
66+
6567
def inverted?(lon_series, lat_series) do
6668
lon_in_lat_range =
6769
Series.and(

‎apps/transport/lib/irve/data_frame.ex‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,14 @@ defmodule Transport.IRVE.DataFrame do
280280
longitude f64 [43.306241]
281281
latitude f64 [-0.332879]
282282
>
283+
284+
The function is resilient to weird values:
285+
iex> Explorer.DataFrame.new([%{coordonneesXY: ""}, %{coordonneesXY: "[hello, 9]"}, %{coordonneesXY: "hello"}]) |> Transport.IRVE.DataFrame.preprocess_xy_coordinates()
286+
#Explorer.DataFrame<
287+
Polars[3 x 2]
288+
longitude f64 [nil, nil, nil]
289+
latitude f64 [nil, 9.0, nil]
290+
>
283291
"""
284292
def preprocess_xy_coordinates(df) do
285293
df

‎apps/transport/lib/irve/validator/data_frame_validation.ex‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,4 +81,24 @@ defmodule Transport.IRVE.Validator.DataFrameValidation do
8181
%{"check_row_valid" => row_valid}
8282
end)
8383
end
84+
85+
@doc """
86+
Adds custom warning columns.
87+
We only apply warnings to valid values to avoid noise.
88+
"""
89+
def setup_computed_warning_columns(%Explorer.DataFrame{} = df) do
90+
df
91+
|> setup_inverted_lon_lat_warning_column()
92+
end
93+
94+
def setup_inverted_lon_lat_warning_column(%Explorer.DataFrame{} = df) do
95+
warning =
96+
Explorer.DataFrame.select(df, ["coordonneesXY"])
97+
# The next line is safe and shouldn’t raise errors even with weird values.
98+
|> Transport.IRVE.Processing.preprocess_coordinates()
99+
|> Transport.IRVE.CoordinateCorrection.inverted?()
100+
|> Explorer.Series.and(df["check_column_coordonneesXY_valid"])
101+
102+
Explorer.DataFrame.put(df, "warning_lon_lat_inverted", warning)
103+
end
84104
end

‎apps/transport/lib/irve/validator/summary.ex‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ defmodule Transport.IRVE.Validator.Summary do
1010
:total_row_count,
1111
:file_level_errors,
1212
:column_errors,
13-
:error_samples
13+
:error_samples,
14+
:warnings,
15+
:warning_samples
1416
]
1517

1618
defstruct [
@@ -20,7 +22,9 @@ defmodule Transport.IRVE.Validator.Summary do
2022
:total_row_count,
2123
:file_level_errors,
2224
:column_errors,
23-
:error_samples
25+
:error_samples,
26+
:warnings,
27+
:warning_samples
2428
]
2529

2630
@doc """
@@ -31,7 +35,10 @@ defmodule Transport.IRVE.Validator.Summary do
3135
def from_result(result) when is_map(result) do
3236
result
3337
|> atomize_keys()
38+
|> Map.put_new(:warnings, %{})
39+
|> Map.put_new(:warning_samples, [])
3440
|> Map.update!(:error_samples, fn samples -> Enum.map(samples, &atomize_keys/1) end)
41+
|> Map.update!(:warning_samples, fn samples -> Enum.map(samples, &atomize_keys/1) end)
3542
|> then(&struct!(__MODULE__, &1))
3643
end
3744

‎apps/transport/lib/irve/validator/validator.ex‎

Lines changed: 45 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ defmodule Transport.IRVE.Validator do
99
df
1010
|> Transport.IRVE.Validator.DataFrameValidation.setup_computed_field_validation_columns(schema)
1111
|> Transport.IRVE.Validator.DataFrameValidation.setup_computed_row_validation_column()
12+
|> Transport.IRVE.Validator.DataFrameValidation.setup_computed_warning_columns()
1213
end
1314

1415
@doc """
@@ -55,6 +56,7 @@ defmodule Transport.IRVE.Validator do
5556
{valid_count, invalid_count} = summarize_total_counts(df)
5657
column_errors = summarize_column_errors(df)
5758
error_samples = error_samples(df, column_errors)
59+
warnings = summarize_warnings(df)
5860

5961
%Transport.IRVE.Validator.Summary{
6062
valid: invalid_count == 0,
@@ -63,7 +65,9 @@ defmodule Transport.IRVE.Validator do
6365
total_row_count: valid_count + invalid_count,
6466
file_level_errors: [],
6567
column_errors: column_errors,
66-
error_samples: error_samples
68+
error_samples: error_samples,
69+
warnings: warnings,
70+
warning_samples: warning_samples(df, warnings)
6771
}
6872
end
6973

@@ -91,10 +95,37 @@ defmodule Transport.IRVE.Validator do
9195
total_row_count: nil,
9296
file_level_errors: [Exception.message(error)],
9397
column_errors: %{},
94-
error_samples: []
98+
error_samples: [],
99+
warnings: %{},
100+
warning_samples: []
95101
}
96102
end
97103

104+
@max_samples_per_group 5
105+
106+
# Maps each warning to the raw input column (not immediately constructed from the warning name)
107+
@warning_value_columns %{"lon_lat_inverted" => "coordonneesXY"}
108+
109+
defp summarize_warnings(df) do
110+
warnings = Explorer.DataFrame.select(df, &String.starts_with?(&1, "warning_"))
111+
112+
warnings
113+
|> Explorer.DataFrame.names()
114+
|> Map.new(fn col -> {String.replace_prefix(col, "warning_", ""), Explorer.Series.sum(warnings[col])} end)
115+
|> Map.reject(fn {_name, count} -> count == 0 end)
116+
end
117+
118+
defp warning_samples(df, warnings) do
119+
warnings
120+
|> Enum.flat_map(fn {warning_name, _count} ->
121+
value_col = Map.fetch!(@warning_value_columns, warning_name)
122+
123+
df
124+
|> Explorer.DataFrame.filter_with(& &1["warning_#{warning_name}"])
125+
|> take_samples(value_col, :warning, warning_name)
126+
end)
127+
end
128+
98129
defp summarize_total_counts(df) do
99130
valid_count = df["check_row_valid"] |> Explorer.Series.sum()
100131
invalid_count = Explorer.DataFrame.n_rows(df) - valid_count
@@ -114,17 +145,19 @@ defmodule Transport.IRVE.Validator do
114145
defp error_samples(df, column_errors) do
115146
column_errors
116147
|> Enum.flat_map(fn {field_name, _error_count} ->
117-
check_col = "check_column_#{field_name}_valid"
118-
119148
df
120-
|> Explorer.DataFrame.filter_with(&(&1[check_col] |> Explorer.Series.not()))
121-
|> Explorer.DataFrame.select(["id_pdc_itinerance", field_name])
122-
# Limit to 5 samples per error column
123-
|> Explorer.DataFrame.head(5)
124-
|> Explorer.DataFrame.to_rows()
125-
|> Enum.map(fn row ->
126-
%{id_pdc_itinerance: row["id_pdc_itinerance"], column: field_name, value: row[field_name]}
127-
end)
149+
|> Explorer.DataFrame.filter_with(&Explorer.Series.not(&1["check_column_#{field_name}_valid"]))
150+
|> take_samples(field_name, :column, field_name)
151+
end)
152+
end
153+
154+
defp take_samples(df, value_col, tag_key, name) do
155+
df
156+
|> Explorer.DataFrame.select(["id_pdc_itinerance", value_col])
157+
|> Explorer.DataFrame.head(@max_samples_per_group)
158+
|> Explorer.DataFrame.to_rows()
159+
|> Enum.map(fn row ->
160+
%{:id_pdc_itinerance => row["id_pdc_itinerance"], tag_key => name, :value => row[value_col]}
128161
end)
129162
end
130163
end

‎apps/transport/lib/transport_web/templates/validation/show_irve_statique.html.heex‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,18 @@
5555
</ul>
5656
<% end %>
5757

58+
<%= unless Enum.empty?(@summary.warnings) do %>
59+
<p>
60+
<span class="icon--validation">⚠️</span>
61+
{dngettext(
62+
"validations",
63+
"%{count} warning detected.",
64+
"%{count} warnings detected.",
65+
Enum.sum(Map.values(@summary.warnings))
66+
)}
67+
</p>
68+
<% end %>
69+
5870
<%= unless Enum.empty?(@summary.file_level_errors) do %>
5971
<h4>{dgettext("validations", "File-level errors")}</h4>
6072
<ul>
@@ -99,6 +111,44 @@
99111
</tbody>
100112
</table>
101113
<% end %>
114+
115+
<%= unless Enum.empty?(@summary.warnings) do %>
116+
<h4>{dgettext("validations", "Warnings")}</h4>
117+
<table class="table">
118+
<thead>
119+
<tr>
120+
<th>{dgettext("validations", "Warning")}</th>
121+
<th>{dgettext("validations", "Rows")}</th>
122+
</tr>
123+
</thead>
124+
<tbody>
125+
<tr :for={{warning, count} <- Enum.sort_by(@summary.warnings, &elem(&1, 0))}>
126+
<td>{warning_label(warning)}</td>
127+
<td>{count}</td>
128+
</tr>
129+
</tbody>
130+
</table>
131+
<% end %>
132+
133+
<%= unless Enum.empty?(@summary.warning_samples) do %>
134+
<h4>{dgettext("validations", "Warning samples")}</h4>
135+
<table class="table">
136+
<thead>
137+
<tr>
138+
<th>id_pdc_itinerance</th>
139+
<th>{dgettext("validations", "Warning")}</th>
140+
<th>{dgettext("validations", "Value")}</th>
141+
</tr>
142+
</thead>
143+
<tbody>
144+
<tr :for={sample <- @summary.warning_samples}>
145+
<td>{sample.id_pdc_itinerance}</td>
146+
<td>{warning_label(sample.warning)}</td>
147+
<td>{sample.value}</td>
148+
</tr>
149+
</tbody>
150+
</table>
151+
<% end %>
102152
</div>
103153
</div>
104154
</div>

‎apps/transport/lib/transport_web/views/validation_view.ex‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@ defmodule TransportWeb.ValidationView do
2424
def has_errors?([]), do: false
2525
def has_errors?(summary) when is_list(summary), do: true
2626

27+
def warning_label("lon_lat_inverted"),
28+
do: dgettext("validations", "Longitude and latitude coordinates inverted")
29+
30+
def warning_label(warning) when is_binary(warning), do: warning
31+
2732
def netex_pagination_links(conn, issues, validation_id, current_category) do
2833
pagination_links(conn, issues, [validation_id],
2934
issues_category: current_category,

‎apps/transport/priv/gettext/en/LC_MESSAGES/validations.po‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -513,3 +513,25 @@ msgstr ""
513513
#, elixir-autogen, elixir-format, fuzzy
514514
msgid "File is too large, must be <%{max_file_size}."
515515
msgstr ""
516+
517+
#, elixir-autogen, elixir-format
518+
msgid "%{count} warning detected."
519+
msgid_plural "%{count} warnings detected."
520+
msgstr[0] ""
521+
msgstr[1] ""
522+
523+
#, elixir-autogen, elixir-format
524+
msgid "Longitude and latitude coordinates inverted"
525+
msgstr ""
526+
527+
#, elixir-autogen, elixir-format
528+
msgid "Rows"
529+
msgstr ""
530+
531+
#, elixir-autogen, elixir-format, fuzzy
532+
msgid "Warning"
533+
msgstr ""
534+
535+
#, elixir-autogen, elixir-format, fuzzy
536+
msgid "Warning samples"
537+
msgstr ""

‎apps/transport/priv/gettext/fr/LC_MESSAGES/validations.po‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -513,3 +513,25 @@ msgstr "Vous pouvez toujours utiliser l'ancien <a href=\"%{url}\" target=\"_blan
513513
#, elixir-autogen, elixir-format
514514
msgid "File is too large, must be <%{max_file_size}."
515515
msgstr "Fichier trop gros, doit peser moins de %{max_file_size}."
516+
517+
#, elixir-autogen, elixir-format
518+
msgid "%{count} warning detected."
519+
msgid_plural "%{count} warnings detected."
520+
msgstr[0] "%{count} avertissement détecté."
521+
msgstr[1] "%{count} avertissements détectés."
522+
523+
#, elixir-autogen, elixir-format
524+
msgid "Longitude and latitude coordinates inverted"
525+
msgstr "Coordonnées de longitude et latitude inversées"
526+
527+
#, elixir-autogen, elixir-format
528+
msgid "Rows"
529+
msgstr "Lignes"
530+
531+
#, elixir-autogen, elixir-format
532+
msgid "Warning"
533+
msgstr "Avertissement"
534+
535+
#, elixir-autogen, elixir-format
536+
msgid "Warning samples"
537+
msgstr "Exemples d'avertissements"

‎apps/transport/priv/gettext/validations.pot‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,3 +511,25 @@ msgstr ""
511511
#, elixir-autogen, elixir-format
512512
msgid "File is too large, must be <%{max_file_size}."
513513
msgstr ""
514+
515+
#, elixir-autogen, elixir-format
516+
msgid "%{count} warning detected."
517+
msgid_plural "%{count} warnings detected."
518+
msgstr[0] ""
519+
msgstr[1] ""
520+
521+
#, elixir-autogen, elixir-format
522+
msgid "Longitude and latitude coordinates inverted"
523+
msgstr ""
524+
525+
#, elixir-autogen, elixir-format
526+
msgid "Rows"
527+
msgstr ""
528+
529+
#, elixir-autogen, elixir-format
530+
msgid "Warning"
531+
msgstr ""
532+
533+
#, elixir-autogen, elixir-format
534+
msgid "Warning samples"
535+
msgstr ""

0 commit comments

Comments
 (0)