Skip to content

Commit 3efdf01

Browse files
ggounotggounot
committed
Use pagination for Dora services import
1 parent 251ca85 commit 3efdf01

4 files changed

Lines changed: 115 additions & 68 deletions

File tree

‎itou/insertion/management/commands/import_structures_and_services.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
)
1818
from itou.utils import constants as global_constants, diff
1919
from itou.utils.apis.data_inclusion import DataInclusionApiV1Client, DataInclusionApiV1ItemsIterator
20-
from itou.utils.apis.dora import DoraAPIClient
20+
from itou.utils.apis.dora import DoraAPIClient, DoraApiItemsIterator
2121
from itou.utils.command import BaseCommand
2222

2323

@@ -456,7 +456,7 @@ def _fill_and_save_service_from_api_data(
456456
def import_services(self, di_client, dora_client):
457457
self.logger.info("Importing services")
458458
disabled_dora_form_di_structures = dora_client.disabled_dora_form_di_structures()
459-
dora_services = dora_client.emplois_services()
459+
dora_services = {"dora--" + item["id"]: item for item in DoraApiItemsIterator(dora_client.emplois_services)}
460460
structures = Structure.objects.only("uid").in_bulk(field_name="uid")
461461

462462
differ = diff.CollectionDiffer(

‎itou/utils/apis/dora.py‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import dataclasses
12
import logging
23

34
import httpx
@@ -10,6 +11,33 @@ class DoraAPIException(Exception):
1011
pass
1112

1213

14+
@dataclasses.dataclass(frozen=True, kw_only=True, slots=True)
15+
class DoraApiPaginatedResponse:
16+
results: list[dict]
17+
count: int
18+
next: str | None
19+
previous: str | None
20+
21+
22+
class DoraApiItemsIterator:
23+
DEFAULT_PAGE = 1
24+
DEFAULT_PAGE_SIZE = 1_000
25+
26+
def __init__(self, client_method, *, page_size=DEFAULT_PAGE_SIZE, params=None):
27+
self._client_method = client_method
28+
self._params = params or {}
29+
self.page_size = page_size
30+
31+
def __iter__(self):
32+
page = self.DEFAULT_PAGE
33+
while True:
34+
response = self._client_method(**{**self._params, "page": page, "page_size": self.page_size})
35+
yield from response.results
36+
if response.next is None:
37+
break
38+
page += 1
39+
40+
1341
class DoraAPIClient:
1442
def __init__(self, base_url: str, token: str):
1543
self.client = httpx.Client(
@@ -42,10 +70,10 @@ def reference_data(self, **params):
4270
return self._request("/emplois/reference-data/", params).json()
4371

4472
def emplois_services(self, **params):
45-
return {
46-
"dora--" + r["id"]: {**r, "uid": "dora--" + r["id"]}
47-
for r in self._request("/emplois/services/", params).json()
48-
}
73+
payload = self._request("/emplois/services/", params).json()
74+
if "page" in params:
75+
return DoraApiPaginatedResponse(**payload)
76+
return payload
4977

5078
def disabled_dora_form_di_structures(self, **params):
5179
return {

‎tests/insertion/__snapshots__/test_import_structures_and_services.ambr‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@
278278
"Change 'is_orientable_with_form' ('True' -> 'False') for count=0 services based on count=2 structures",
279279
'Importing services',
280280
'HTTP Request: GET https://dora-api/api/emplois/disabled-dora-form-di-structures/ "HTTP/1.1 200 OK"',
281-
'HTTP Request: GET https://dora-api/api/emplois/services/ "HTTP/1.1 200 OK"',
281+
'HTTP Request: GET https://dora-api/api/emplois/services/?page=1&page_size=1000 "HTTP/1.1 200 OK"',
282282
'HTTP Request: GET https://api.data.inclusion.gouv.fr/api/v1/services?page=1&size=5000 "HTTP/1.1 200 OK"',
283283
"ADDED item key=('emplois-de-linclusion--null',) data=['updated_on']",
284284
"ADDED item key=('emplois-de-linclusion--empty',) data=['updated_on']",
@@ -321,7 +321,7 @@
321321
"Change 'is_orientable_with_form' ('True' -> 'False') for count=0 services based on count=2 structures",
322322
'Importing services',
323323
'HTTP Request: GET https://dora-api/api/emplois/disabled-dora-form-di-structures/ "HTTP/1.1 200 OK"',
324-
'HTTP Request: GET https://dora-api/api/emplois/services/ "HTTP/1.1 200 OK"',
324+
'HTTP Request: GET https://dora-api/api/emplois/services/?page=1&page_size=1000 "HTTP/1.1 200 OK"',
325325
'HTTP Request: GET https://api.data.inclusion.gouv.fr/api/v1/services?page=1&size=5000 "HTTP/1.1 200 OK"',
326326
'SUMMARY Service current_collection_keys_count=4 comparative_collection_keys_count=4 common_keys_count=4 added_keys_count=0 updated_keys_count=0 removed_keys_count=0',
327327
])
@@ -1943,7 +1943,7 @@
19431943
"Change 'is_orientable_with_form' ('True' -> 'False') for count=0 services based on count=2 structures",
19441944
'Importing services',
19451945
'HTTP Request: GET https://dora-api/api/emplois/disabled-dora-form-di-structures/ "HTTP/1.1 200 OK"',
1946-
'HTTP Request: GET https://dora-api/api/emplois/services/ "HTTP/1.1 200 OK"',
1946+
'HTTP Request: GET https://dora-api/api/emplois/services/?page=1&page_size=1000 "HTTP/1.1 200 OK"',
19471947
'HTTP Request: GET https://api.data.inclusion.gouv.fr/api/v1/services?page=1&size=5000 "HTTP/1.1 200 OK"',
19481948
"ADDED item key=('emplois-de-linclusion--null',) data=['updated_on']",
19491949
"ADDED item key=('emplois-de-linclusion--empty',) data=['updated_on']",

‎tests/insertion/api_mocks/dora/services.json‎

Lines changed: 78 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,78 @@
1-
[
2-
{
3-
"id": "46f7ea19-c97b-4f45-90a9-027b44cad927",
4-
"short_desc": "Les emplois de l'inclusion est un service numérique de mise en relation d'employeurs solidaires avec des candidats éloignés de l'emploi par le biais de tiers (prescripteurs habilités, orienteurs) ou en auto-prescription.",
5-
"recurrence": "",
6-
"funding_labels": [],
7-
"coach_orientation_modes": ["autre", "formulaire-dora"],
8-
"coach_orientation_modes_other": "S'inscrire et postuler sur https://emplois.inclusion.beta.gouv.fr/",
9-
"coach_orientation_modes_external_form_link": "",
10-
"coach_orientation_modes_external_form_link_text": "Orienter votre bénéficiaire",
11-
"beneficiaries_access_modes": ["autre"],
12-
"beneficiaries_access_modes_other": "S'inscrire et postuler sur https://emplois.inclusion.beta.gouv.fr/",
13-
"beneficiaries_access_modes_external_form_link": "",
14-
"beneficiaries_access_modes_external_form_link_text": "Faire une demande",
15-
"forms": [
16-
{
17-
"name": "cc4e1fbc-533b-46e2-8b33-bc31c33c9ffd/2025-03-13_15h35_21.png",
18-
"url": "http://dora.url/cc4e1fbc-533b-46e2-8b33-bc31c33c9ffd/2025-03-13_15h35_21.png"
19-
},
20-
{
21-
"name": "cc4e1fbc-533b-46e2-8b33-bc31c33c9ffd/2025-03-13_15h34_58.png",
22-
"url": "http://dora.url/cc4e1fbc-533b-46e2-8b33-bc31c33c9ffd/2025-03-13_15h34_58.png"
23-
}
24-
],
25-
"online_form": "",
26-
"access_conditions": ["Résident QPV / ZRR", "Personne non imposable"],
27-
"credentials": ["Contrat d'apprentissage"],
28-
"is_orientable_with_form": true,
29-
"contact_name": "Z... H...",
30-
"contact_phone": "9690430172",
31-
"contact_email": "contact-emplois@yopmail.com",
32-
"is_contact_info_public": false,
33-
"average_orientation_response_delay_days": 16
34-
},
35-
{
36-
"id": "b6f651e2-56d7-4ffa-a1c6-ae7295089a9e",
37-
"short_desc": "Immersion Facilitée donne accès à des entreprises qui accueillent en immersion et simplifie les démarches de conventionnement de l'immersion.",
38-
"recurrence": "",
39-
"funding_labels": [],
40-
"coach_orientation_modes": ["autre"],
41-
"coach_orientation_modes_other": "https://immersion-facile.beta.gouv.fr/",
42-
"coach_orientation_modes_external_form_link": "",
43-
"coach_orientation_modes_external_form_link_text": "",
44-
"beneficiaries_access_modes": ["autre"],
45-
"beneficiaries_access_modes_other": "https://immersion-facile.beta.gouv.fr/",
46-
"beneficiaries_access_modes_external_form_link": "",
47-
"beneficiaries_access_modes_external_form_link_text": "",
48-
"forms": [],
49-
"online_form": "https://immersion-facile.beta.gouv.fr/",
50-
"access_conditions": [],
51-
"credentials": [],
52-
"is_orientable_with_form": false,
53-
"contact_name": "Ansel Hauck",
54-
"contact_phone": "9247096169",
55-
"contact_email": "gianni@example.com",
56-
"is_contact_info_public": true,
57-
"average_orientation_response_delay_days": null
58-
}
59-
]
1+
{
2+
"count": 2,
3+
"next": null,
4+
"previous": null,
5+
"results": [
6+
{
7+
"id": "46f7ea19-c97b-4f45-90a9-027b44cad927",
8+
"short_desc": "Les emplois de l'inclusion est un service numérique de mise en relation d'employeurs solidaires avec des candidats éloignés de l'emploi par le biais de tiers (prescripteurs habilités, orienteurs) ou en auto-prescription.",
9+
"recurrence": "",
10+
"funding_labels": [],
11+
"coach_orientation_modes": [
12+
"autre",
13+
"formulaire-dora"
14+
],
15+
"coach_orientation_modes_other": "S'inscrire et postuler sur https://emplois.inclusion.beta.gouv.fr/",
16+
"coach_orientation_modes_external_form_link": "",
17+
"coach_orientation_modes_external_form_link_text": "Orienter votre bénéficiaire",
18+
"beneficiaries_access_modes": [
19+
"autre"
20+
],
21+
"beneficiaries_access_modes_other": "S'inscrire et postuler sur https://emplois.inclusion.beta.gouv.fr/",
22+
"beneficiaries_access_modes_external_form_link": "",
23+
"beneficiaries_access_modes_external_form_link_text": "Faire une demande",
24+
"forms": [
25+
{
26+
"name": "cc4e1fbc-533b-46e2-8b33-bc31c33c9ffd/2025-03-13_15h35_21.png",
27+
"url": "http://dora.url/cc4e1fbc-533b-46e2-8b33-bc31c33c9ffd/2025-03-13_15h35_21.png"
28+
},
29+
{
30+
"name": "cc4e1fbc-533b-46e2-8b33-bc31c33c9ffd/2025-03-13_15h34_58.png",
31+
"url": "http://dora.url/cc4e1fbc-533b-46e2-8b33-bc31c33c9ffd/2025-03-13_15h34_58.png"
32+
}
33+
],
34+
"online_form": "",
35+
"access_conditions": [
36+
"Résident QPV / ZRR",
37+
"Personne non imposable"
38+
],
39+
"credentials": [
40+
"Contrat d'apprentissage"
41+
],
42+
"is_orientable_with_form": true,
43+
"contact_name": "Z... H...",
44+
"contact_phone": "9690430172",
45+
"contact_email": "contact-emplois@yopmail.com",
46+
"is_contact_info_public": false,
47+
"average_orientation_response_delay_days": 16
48+
},
49+
{
50+
"id": "b6f651e2-56d7-4ffa-a1c6-ae7295089a9e",
51+
"short_desc": "Immersion Facilitée donne accès à des entreprises qui accueillent en immersion et simplifie les démarches de conventionnement de l'immersion.",
52+
"recurrence": "",
53+
"funding_labels": [],
54+
"coach_orientation_modes": [
55+
"autre"
56+
],
57+
"coach_orientation_modes_other": "https://immersion-facile.beta.gouv.fr/",
58+
"coach_orientation_modes_external_form_link": "",
59+
"coach_orientation_modes_external_form_link_text": "",
60+
"beneficiaries_access_modes": [
61+
"autre"
62+
],
63+
"beneficiaries_access_modes_other": "https://immersion-facile.beta.gouv.fr/",
64+
"beneficiaries_access_modes_external_form_link": "",
65+
"beneficiaries_access_modes_external_form_link_text": "",
66+
"forms": [],
67+
"online_form": "https://immersion-facile.beta.gouv.fr/",
68+
"access_conditions": [],
69+
"credentials": [],
70+
"is_orientable_with_form": false,
71+
"contact_name": "Ansel Hauck",
72+
"contact_phone": "9247096169",
73+
"contact_email": "gianni@example.com",
74+
"is_contact_info_public": true,
75+
"average_orientation_response_delay_days": null
76+
}
77+
]
78+
}

0 commit comments

Comments
 (0)