Skip to content

Commit 10af5dc

Browse files
authored
Redressement des coordonnées dans la consolidation IRVE (#5535)
1 parent efceeb3 commit 10af5dc

9 files changed

Lines changed: 120 additions & 8 deletions

apps/transport/lib/db/irve_valid_pdc.ex

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ defmodule DB.IRVEValidPDC do
5656
field(:cable_t2_attache, :boolean)
5757
field(:longitude, :decimal, null: false)
5858
field(:latitude, :decimal, null: false)
59+
field(:consolidated_is_lon_lat_correct, :boolean, null: false)
5960

6061
timestamps(type: :utc_datetime_usec)
6162
end
@@ -75,5 +76,7 @@ defmodule DB.IRVEValidPDC do
7576
end
7677

7778
defp valid_fields,
78-
do: Transport.IRVE.StaticIRVESchema.field_names_list() ++ ["id", "irve_valid_file_id", "longitude", "latitude"]
79+
do:
80+
Transport.IRVE.StaticIRVESchema.field_names_list() ++
81+
["id", "irve_valid_file_id", "longitude", "latitude", "consolidated_is_lon_lat_correct"]
7982
end
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
defmodule Transport.IRVE.CoordinateCorrection do
2+
@moduledoc """
3+
Detects and corrects lon/lat coordinate inversions in IRVE data.
4+
5+
Some producers submit `coordonneesXY` as `[latitude, longitude]` instead of the
6+
schema-required `[longitude, latitude]`. Detection is based on a bounding box for
7+
metropolitan France + Corsica (lon ∈ [-5.5, 9.7], lat ∈ [41.0, 51.5]); see `inverted?/2`.
8+
"""
9+
10+
@metro_lon_min -5.5
11+
@metro_lon_max 9.7
12+
@metro_lat_min 41.0
13+
@metro_lat_max 51.5
14+
15+
require Explorer.DataFrame, as: DF
16+
alias Explorer.Series
17+
18+
@doc """
19+
Corrects inverted coordinates in a DataFrame with `longitude` and `latitude` float
20+
columns. Adds `consolidated_is_lon_lat_correct` (`false` = row was swapped).
21+
22+
iex> df = Explorer.DataFrame.new(longitude: [2.35, 48.85, 55.4], latitude: [48.85, 2.35, -21.1])
23+
iex> r = Transport.IRVE.CoordinateCorrection.detect_and_correct(df)
24+
iex> Explorer.Series.to_list(r["longitude"])
25+
[2.35, 2.35, 55.4]
26+
iex> Explorer.Series.to_list(r["latitude"])
27+
[48.85, 48.85, -21.1]
28+
iex> Explorer.Series.to_list(r["consolidated_is_lon_lat_correct"])
29+
[true, false, true]
30+
31+
"""
32+
def detect_and_correct(%DF{} = df) do
33+
DF.mutate_with(df, fn df ->
34+
lon = df["longitude"]
35+
lat = df["latitude"]
36+
inverted = inverted?(lon, lat)
37+
38+
%{
39+
longitude: Series.select(inverted, lat, lon),
40+
latitude: Series.select(inverted, lon, lat),
41+
consolidated_is_lon_lat_correct: Series.not(inverted)
42+
}
43+
end)
44+
end
45+
46+
@doc """
47+
Returns a boolean Series: `true` where `lon_series` ∈ [#{@metro_lat_min}, #{@metro_lat_max}]
48+
AND `lat_series` ∈ [#{@metro_lon_min}, #{@metro_lon_max}].
49+
50+
Test cases:
51+
52+
1. Metro France, correct — lon 2.35 (Paris), not in lat range → `false`
53+
2. Metro France, inverted — lon 48.85 is in [41, 51.5] and lat 2.35 is in [-5.5, 9.7] → `true`
54+
3. Mayotte, correct — lon 45.1 falls in the lat range [41, 51.5], but lat −12.8 is
55+
below −5.5, so both conditions are not met → `false`
56+
4. Corsica, inverted — edge case: lat 9.56 is just above the 9.5 ceiling one might
57+
naively use; the 9.7 upper bound is required to catch it → `true`
58+
59+
iex> lon = Explorer.Series.from_list([2.35, 48.85, 45.1, 42.4])
60+
iex> lat = Explorer.Series.from_list([48.85, 2.35, -12.8, 9.56])
61+
iex> Explorer.Series.to_list(Transport.IRVE.CoordinateCorrection.inverted?(lon, lat))
62+
[false, true, false, true]
63+
64+
"""
65+
def inverted?(lon_series, lat_series) do
66+
lon_in_lat_range =
67+
Series.and(
68+
Series.greater_equal(lon_series, @metro_lat_min),
69+
Series.less_equal(lon_series, @metro_lat_max)
70+
)
71+
72+
lat_in_lon_range =
73+
Series.and(
74+
Series.greater_equal(lat_series, @metro_lon_min),
75+
Series.less_equal(lat_series, @metro_lon_max)
76+
)
77+
78+
Series.and(lon_in_lat_range, lat_in_lon_range)
79+
end
80+
end

apps/transport/lib/irve/database_exporter.ex

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ defmodule Transport.IRVE.DatabaseExporter do
5858
def database_field_list do
5959
Transport.IRVE.StaticIRVESchema.field_names_list()
6060
|> Enum.reject(&(&1 == "coordonneesXY"))
61-
|> Enum.concat(["longitude", "latitude"])
61+
|> Enum.concat(["longitude", "latitude", "consolidated_is_lon_lat_correct"])
6262
end
6363

6464
def additional_file_field_list do
@@ -78,7 +78,7 @@ defmodule Transport.IRVE.DatabaseExporter do
7878
"""
7979
def export_field_list do
8080
Transport.IRVE.StaticIRVESchema.field_names_list()
81-
|> Enum.concat(["consolidated_longitude", "consolidated_latitude"])
81+
|> Enum.concat(["consolidated_longitude", "consolidated_latitude", "consolidated_is_lon_lat_correct"])
8282
|> Enum.concat(additional_file_field_list())
8383
end
8484
end

apps/transport/lib/irve/processing.ex

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ defmodule Transport.IRVE.Processing do
1212
|> convert_to_dataframe!()
1313
|> add_missing_optional_columns()
1414
|> preprocess_coordinates()
15+
|> Transport.IRVE.CoordinateCorrection.detect_and_correct()
1516
|> preprocess_boolean_fields()
1617
|> select_fields()
1718
end
@@ -88,7 +89,7 @@ defmodule Transport.IRVE.Processing do
8889
|> Explorer.DataFrame.select(
8990
(Transport.IRVE.StaticIRVESchema.field_names_list() --
9091
["coordonneesXY", "cable_t2_attache"]) ++
91-
["longitude", "latitude"]
92+
["longitude", "latitude", "consolidated_is_lon_lat_correct"]
9293
)
9394
end
9495
end
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
defmodule Transport.Repo.Migrations.AddConsolidatedIsLonLatCorrectToIrveValidPdc do
2+
use Ecto.Migration
3+
4+
def change do
5+
alter table(:irve_valid_pdc) do
6+
add(:consolidated_is_lon_lat_correct, :boolean, null: false, default: false)
7+
end
8+
end
9+
end

apps/transport/test/db/irve_valid_pdc_test.exs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,14 @@ defmodule DB.IRVEValidPDCTest do
1212

1313
assert official_schema_fields -- ["coordonneesXY"] ==
1414
ecto_schema_fields --
15-
["id", "irve_valid_file_id", "inserted_at", "updated_at", "longitude", "latitude"]
15+
[
16+
"id",
17+
"irve_valid_file_id",
18+
"inserted_at",
19+
"updated_at",
20+
"longitude",
21+
"latitude",
22+
"consolidated_is_lon_lat_correct"
23+
]
1624
end
1725
end

apps/transport/test/transport/irve/consolidation_test.exs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ defmodule Transport.IRVE.ConsolidationTest do
129129
|> Map.put("datagouv_organization_or_owner", "the-org")
130130
|> Map.put("datagouv_last_modified", "2024-02-29T07:43:59.000000+0000")
131131
|> Map.put("deduplication_status", "unique")
132+
|> Map.put("consolidated_is_lon_lat_correct", true)
132133
]
133134
|> Explorer.DataFrame.new()
134135
# Use the same column order as in the actual implementation
@@ -137,6 +138,7 @@ defmodule Transport.IRVE.ConsolidationTest do
137138
|> Enum.concat([
138139
"consolidated_longitude",
139140
"consolidated_latitude",
141+
"consolidated_is_lon_lat_correct",
140142
"datagouv_dataset_id",
141143
"datagouv_resource_id",
142144
"dataset_title",
@@ -158,7 +160,7 @@ defmodule Transport.IRVE.ConsolidationTest do
158160
start_path: "consolidation_transport_avec_doublons_irve_statique_#{date}",
159161
bucket: bucket_name,
160162
acl: :private,
161-
file_content: "6c76cfc5918ead5a10e36f39e34995370184c47801c7568e5b7b2dc2a2a75714"
163+
file_content: sha256_of(consolidation_content)
162164
)
163165

164166
Transport.Test.S3TestUtils.s3_mocks_remote_copy_file(
@@ -185,7 +187,7 @@ defmodule Transport.IRVE.ConsolidationTest do
185187
start_path: "consolidation_transport_irve_statique_#{date}",
186188
bucket: bucket_name,
187189
acl: :private,
188-
file_content: "6c76cfc5918ead5a10e36f39e34995370184c47801c7568e5b7b2dc2a2a75714"
190+
file_content: sha256_of(consolidation_content)
189191
)
190192

191193
Transport.Test.S3TestUtils.s3_mocks_remote_copy_file(
@@ -280,4 +282,8 @@ defmodule Transport.IRVE.ConsolidationTest do
280282
File.write!(path, body)
281283
%Req.Response{status: 200, body: File.stream!(path)}
282284
end
285+
286+
defp sha256_of(content) do
287+
:crypto.hash(:sha256, content) |> Base.encode16() |> String.downcase()
288+
end
283289
end
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
defmodule Transport.IRVE.CoordinateCorrectionTest do
2+
use ExUnit.Case, async: true
3+
doctest Transport.IRVE.CoordinateCorrection, import: true
4+
end

apps/transport/test/transport/irve/processing_test.exs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ defmodule Transport.IRVE.ProcessingTest do
6666
# This was added and coordonneesXY removed
6767
"longitude" => -0.799141,
6868
# Same
69-
"latitude" => 45.91914
69+
"latitude" => 45.91914,
70+
"consolidated_is_lon_lat_correct" => true
7071
}
7172
]
7273
end

0 commit comments

Comments
 (0)