Merge branch 'main' into feat/presentation/elu

Author: ThomGram

.github/workflows/pre-commit-tests.yaml

@@ -37,4 +37,4 @@ jobs:
         poetry run pytest -v
     - name: Verify tests results
       if: ${{ failure() }}
-      run: exit 1
+      run: exit 1

README.md

@@ -80,6 +80,20 @@ Pour comprendre en détail comment ça fonctionne :
 
 [Comprendre les Extracteurs de données](./docs/extract.md)
 
+## Chargement
+
+Après avoir extrait les données, il est nécessaire de les charger en base de donnée avant de lancer la transformation dbt.
+
+Le script “load“ se lance exactement comme le script extract ci-dessus, avec les mêmes arguments :
+
+```bash
+# Extraire tous les datasets source du domaine "geographical_references"
+poetry run bin/odis.py load --domain geographical_references
+
+# Extraire seulement les datasets "regions" et "departements du domaine "geographical_references"
+poetry run bin/odis.py load --sources geographical_references.regions, geographical_references.departements
+```
+
 ## Sonder les sources disponibles
 
 L’option “explain” permet de voir facilement comment les API, Domaines et Sources sont définis dans la configuration. Si l’option “explain” est passée, le script n’extrait aucune donnée mais montre seulement les infos sur les configurations demandées.

bin/odis.py

@@ -152,7 +152,7 @@ async def extract_data_sources(
         )
 
         for exc in tasks_exceptions:
-            logger.error(f"Error: {exc}")
+            logger.exception(f"Error during extraction: {exc}", exc_info=exc)
 
         # exit with a non-zero status code
         # to indicate that there was an error

common/utils/http/async_client.py

@@ -19,26 +19,30 @@ class AsyncHttpClient(HttpClient):
     """
 
     _session: aiohttp.ClientSession
+    _timeout: aiohttp.ClientTimeout
 
-    def __init__(self, max_connections: int = 100):
+    def __init__(self, max_connections: int = 100, timeout: int = 1200):
         """
         Args:
             max_connections (int, optional): The max number of concurrent connections.
                 Defaults to 100.
+            timeout (int, optional): The timeout in seconds for each individual HTTP request.
+                Defaults to 1200 (20 minutes).
         """
 
         conn = aiohttp.TCPConnector(limit=max_connections)
-        self._session = aiohttp.ClientSession(connector=conn)
+        self._timeout = aiohttp.ClientTimeout(total=timeout)
+        self._session = aiohttp.ClientSession(connector=conn, timeout=self._timeout)
 
         logger.debug(
-            f"AsyncHttpClient initialized with max_connections={max_connections}"
+            f"AsyncHttpClient initialized with max_connections={max_connections}, timeout={timeout}s"
         )
 
     @retry(
         retry=retry_if_exception_type(aiohttp.ClientError),
-        stop=(stop_after_delay(180) | stop_after_attempt(3)),
+        stop=(stop_after_delay(2400) | stop_after_attempt(3)),
         before=before_log(logger, logging.DEBUG),
-        reraise=True,  # re-raise the last exception
+        reraise=True,  
     )
     async def get(
         self,

common/utils/source_extractors.py

@@ -98,11 +98,11 @@ async def download_page(self, url: str) -> ExtractionResult:
             # if not given in API response, BUT if no next page could be derived, then true
             # else: false
             is_last_key = self.model.response_map.get("is_last")
-            is_last = (
-                jmespath.search(is_last_key, payload)
-                if is_last_key
-                else (next_url is None)
-            )
+            if is_last_key:
+                is_last_from_response = jmespath.search(is_last_key, payload)
+                is_last = is_last_from_response if is_last_from_response is not None else (next_url is None)
+            else:
+                is_last = (next_url is None)
 
             # If all went well, success = true
             success = True

datasources.yaml

@@ -288,7 +288,7 @@ domains:
       API: GEO.api.gouv.fr
       description: Référentiel géographique GEO - niveau commune
       type: JsonExtractor
-      endpoint: /communes?fields=code,nom,population,departement,region,centre
+      endpoint: /communes?fields=code,nom,population,departement,region,centre,epci
       format: json
 
     arrondissements:
@@ -315,6 +315,15 @@ domains:
         name: excelfile_dfload
         type: notebook
 
+    geocodes_passage_annuel:
+      API: Opendatasoft
+      description: Table de passage annuelle 2025, équivalence codes géographiques 2025 et année précédente d'établissement de codes (2023)
+      type: FileExtractor
+      endpoint: /catalog/datasets/georef-france-matching-code/exports/csv?use_labels=true
+      format: csv
+      load_params:
+        separator: ;
+
   accueil:
 
     cada_cph_huda:
@@ -338,14 +347,13 @@ domains:
           maxResult: 10000
           # TIME_PERIOD: 2021
           startPeriod: "2009-01-01"
-          endPeriod: "2025-01-01"
+          endPeriod: "2025-10-01"
           GEO: # géographies
             - COM # communes
             - DEP # départements
             - REG # régions
           RP_MEASURE: DWELLINGS # nombre de logements
           L_STAY: _T
-          TOH: _T
           CARS: _T
           NOR: _T
           TSH: # statut d'occupation du logement
@@ -379,11 +387,11 @@ domains:
         format: json
         extract_params:
           maxResult: 10000
-          TIME_PERIOD: [2010,2015,2021,2025]
+          startPeriod: "2009-01-01"
+          endPeriod: "2025-10-01"
           GEO: ["COM", "DEP", "REG"]
           RP_MEASURE: DWELLINGS # nombre de logements
           L_STAY: _T
-          TOH: _T
           CARS: _T
           NOR: _T
           TSH: # statut d'occupation du logement
@@ -412,11 +420,11 @@ domains:
         format: json
         extract_params:
           maxResult: 10000
-          TIME_PERIOD: [2010,2015,2021,2025]
+          startPeriod: "2009-01-01"
+          endPeriod: "2025-10-01"
           GEO: ["COM", "DEP", "REG"]
           RP_MEASURE: DWELLINGS
           L_STAY: _T
-          TOH: _T
           CARS: _T
           NOR: _T
           TSH:
@@ -442,11 +450,11 @@ domains:
         format: json
         extract_params:
           maxResult: 10000
-          TIME_PERIOD: [2010,2015,2021,2025]
+          startPeriod: "2009-01-01"
+          endPeriod: "2025-10-01"
           GEO: ["COM", "DEP", "REG"]
           RP_MEASURE: DWELLINGS
           L_STAY: _T
-          TOH: _T
           CARS: _T
           NOR: _T
           TSH:
@@ -472,11 +480,11 @@ domains:
         format: json
         extract_params:
           maxResult: 10000
-          TIME_PERIOD: [2010,2015,2021,2025]
+          startPeriod: "2009-01-01"
+          endPeriod: "2025-10-01"
           GEO: ["COM", "DEP", "REG"]
           RP_MEASURE: DWELLINGS_ROOMS # nombre de pièces
           L_STAY: _T
-          TOH: _T
           CARS: _T
           NOR: _T
           TSH: # statut d'occupation du logement
@@ -557,7 +565,8 @@ domains:
         format: json
         extract_params:
           maxResult: 10000
-          TIME_PERIOD: 2023
+          startPeriod: "2009-01-01"
+          endPeriod: "2025-10-01"
           GEO: ["COM","REG","DEP"]
           FACILITY_TYPE: ["A122", "A128", "A203", "A206", "A208","A304","A501","A503", "A504"]
         response_map:
@@ -569,7 +578,6 @@ domains:
 
   population:
 
-
       by_age:
         API: INSEE.Melodi
         description: Répartition de la population par tranches d'âge
@@ -587,42 +595,21 @@ domains:
 
       by_age_gender:
         API: INSEE.Melodi
-        description: Répartition de la population par tranches d'âge et par genre (M/F, Cis/Trans)
+        description: Répartition de la population par tranches d'âge et par genre
         type: MelodiExtractor
-        endpoint: /data/DS_ESTIMATION_POPULATION # Dataset des estimations de population de l'INSEE
+        endpoint: /data/DS_RP_POPULATION_PRINC # recensement données principales
         format: json
         extract_params:
           maxResult: 10000
-          SEX:
-            - M # Hommes (all)
-            - M_cis # Hommes cis
-            - F # Femmes (all)
-            - F_cis # Femmes cis
-            - TS # Personnes Trans
-            - _Z # 'Non Applicable' 
-          EP_MEASURE: POP # Prendre uniquement la mesure de population
+          TIME_PERIOD: 2022
+          SEX: ["M", "F", "_T"]
+          GEO: ["COM", "DEP", "REG"]
+          RP_MEASURE: POP
         response_map:
           data: observations
           next: paging.next
           is_last: paging.isLast
 
-      # repartition_sexe_age:
-      #   API: INSEE.Melodi
-      #   description: repartition de la population par sexe et par tranche d'age
-      #   type: MelodiExtractor
-      #   endpoint: /data/DS_RP_POPULATION_PRINC
-      #   format: json
-      #   extract_params:
-      #     maxResult: 10000
-      #     TIME_PERIOD: 2021
-      #     GEO: ["COM","DEP","REG"]
-      #     SEX: ["F","M"]
-      #     AGE: ["Y_LT15","Y15T24","Y25T39","Y40T54","Y55T64","Y65T79","Y_GE80"]
-      #   response_map:
-      #     data: observations
-      #     next: paging.next
-      #     is_last: paging.isLast
-      
       population_superficie:
         API: INSEE.Melodi
         description: recuperation de la population et de la superficie pour calcul de densite
@@ -631,14 +618,31 @@ domains:
         format: json
         extract_params:
           maxResult: 10000
-          TIME_PERIOD: 2021
+          TIME_PERIOD: 2022
           GEO: ["COM","DEP","REG"]
           RP_MEASURE: ["POP","SUP"]
           OCS: "_T"
         response_map:
           data: observations
           next: paging.next
           is_last: paging.isLast
+
+      population_totale:
+        API: INSEE.Melodi
+        description: recuperation des populations communale et totale des communes
+        type: MelodiExtractor
+        endpoint: /data/DS_POPULATIONS_REFERENCE
+        format: json
+        extract_params:
+          maxResult: 10000
+          TIME_PERIOD: 2022
+          GEO: ["COM"]
+          POPREF_MEASURE: ["PMUN", "PTOT"]
+          page: 1
+        response_map:
+          data: observations
+          next: paging.next
+          is_last: paging.isLast
 
       categorie_socio_pro:
         API: INSEE.Melodi
@@ -648,12 +652,12 @@ domains:
         format: json
         extract_params:
           maxResult: 10000
-          TIME_PERIOD: 2021
+          TIME_PERIOD: 2022
           GEO: ["COM","DEP","REG"]
           SEX: "_T"
           RP_MEASURE: "POP"
-          AGE: "Y_GE15"
-          PCS: ["1","2","3","4","5","6","7","9"]
+          AGE: "Y_GE15" # 15 years old or more
+          PCS: ["1", "2", "3", "4", "5", "6", "7", "9", "_T"] # 8 was excluded (students), added it back just in case
         response_map:
           data: observations
           next: paging.next