✨(ingestion) load offer for vacancy webhooks (#592)

* ✨(ingestion) load offer for vacancy webhooks

* ♻️(ingestion) refactor code and handle PR comments

* ✅(tooling) refactor ingestion GitHub Actions workflow

* 🔧(tooling) trigger CodeQL scan

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: AntoineAugusti

.github/workflows/ingestion.yml

@@ -0,0 +1,56 @@
+name: Ingestion
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+    types: [opened, synchronize, reopened, edited]
+    paths:
+      - "src/ingestion/**"
+      - ".github/workflows/ingestion.yml"
+      - ".github/actions/**"
+
+permissions:
+  contents: read
+
+jobs:
+  build-ingestion:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/setup-uv-python
+        with:
+          service-path: src/ingestion
+
+  lint-ingestion:
+    needs: build-ingestion
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./src/ingestion
+    steps:
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/setup-uv-python
+        with:
+          service-path: src/ingestion
+      - name: Lint with Ruff
+        run: uv run ruff check .
+      - name: Lint format with Ruff
+        run: uv run ruff format --check .
+      - name: Lint with MyPy
+        run: uv run mypy .
+
+  test-ingestion:
+    needs: build-ingestion
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./src/ingestion
+    steps:
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/setup-uv-python
+        with:
+          service-path: src/ingestion
+      - name: Test with pytest
+        run: uv run pytest --cov=. --cov-report=term-missing --cov-fail-under=95

Makefile

@@ -404,10 +404,23 @@ test-cov-ocr: ## run ocr tests with detailed HTML coverage report
 	@open src/ocr/tests/cov_html/index.html
 .PHONY: test-cov-ocr
 
+test-cov-ingestion: ## run ingestion tests with detailed HTML coverage report
+	@echo 'test:cov-ingestion started…'
+	@echo 'Generating detailed HTML coverage report for ingestion…'
+	@if [ ! -f "src/ingestion/tests/cov_html/index.html" ]; then \
+		echo '⚠️  Coverage report not found. Creating directory structure...'; \
+		mkdir -p src/ingestion/tests/cov_html; \
+	fi
+	$(INGESTION_UV) pytest --cov=. --cov-report=html:tests/cov_html --cov-report=term-missing $(ARGS)
+	@echo '✅ Coverage report generated in src/ingestion/tests/cov_html/'
+	@open src/ingestion/tests/cov_html/index.html
+.PHONY: test-cov-ingestion
+
 test-cov: ## run tests with detailed HTML coverage report for all services
 test-cov: \
   test-cov-web \
-  test-cov-ocr
+  test-cov-ocr \
+  test-cov-ingestion
 .PHONY: test-cov
 
 ## MANAGE docker services

env.d/ingestion-example

@@ -1,5 +1,6 @@
 TALENTSOFT_CLIENT_ID=
 TALENTSOFT_CLIENT_SECRET=
+TALENTSOFT_BASE_URL=
 
 WEB_BASE_URL=
 WEB_API_KEY=

src/ingestion/api/config.py

@@ -13,6 +13,7 @@ class Settings(BaseSettings):
 
     talentsoft_client_id: str | None = None
     talentsoft_client_secret: str | None = None
+    talentsoft_base_url: str | None = None
 
     web_base_url: str | None = None
     web_api_key: str | None = None
@@ -27,6 +28,10 @@ class TestSettings(Settings):
     sentry_profiles_sample_rate: float | None = 0.0
     sentry_traces_sample_rate: float | None = 0.0
 
+    talentsoft_client_id: str | None = None
+    talentsoft_client_secret: str | None = None
+    talentsoft_base_url: str | None = None
+
     web_base_url: str | None = None
     web_api_key: str | None = None
 

src/ingestion/api/dependencies.py

@@ -1,10 +1,18 @@
+import logging
 from collections.abc import AsyncGenerator
 
 import httpx
-from fastapi import Depends, HTTPException
+from fastapi import Depends
 
 from api.config import Settings, get_settings
 from application.use_cases.archive_offer import ArchiveOfferUseCase
+from application.use_cases.load_offer_details import LoadOfferDetailsUseCase
+from infrastructure.external_gateways.talentsoft_client import (
+    TalentsoftConfig,
+    TalentsoftFrontClient,
+)
+
+logger = logging.getLogger(__name__)
 
 
 async def get_http_client() -> AsyncGenerator[httpx.AsyncClient, None]:
@@ -15,11 +23,30 @@ async def get_http_client() -> AsyncGenerator[httpx.AsyncClient, None]:
 def get_archive_offer_use_case(
     settings: Settings = Depends(get_settings),
     client: httpx.AsyncClient = Depends(get_http_client),
-) -> ArchiveOfferUseCase:
+) -> ArchiveOfferUseCase | None:
     if not settings.web_base_url or not settings.web_api_key:
-        raise HTTPException(status_code=500, detail="Web service not configured")
+        return None
     return ArchiveOfferUseCase(
         client=client,
         web_base_url=settings.web_base_url,
         web_api_key=settings.web_api_key,
     )
+
+
+async def get_load_offer_details_use_case(
+    settings: Settings = Depends(get_settings),
+) -> AsyncGenerator[LoadOfferDetailsUseCase | None, None]:
+    if (
+        not settings.talentsoft_client_id
+        or not settings.talentsoft_client_secret
+        or not settings.talentsoft_base_url
+    ):
+        yield None
+    else:
+        config = TalentsoftConfig(
+            base_url=settings.talentsoft_base_url,
+            client_id=settings.talentsoft_client_id,
+            client_secret=settings.talentsoft_client_secret,
+        )
+        async with TalentsoftFrontClient(config=config, logger=logger) as client:
+            yield LoadOfferDetailsUseCase(talentsoft_client=client)

src/ingestion/api/routes.py

@@ -1,20 +1,24 @@
 import logging
 
-from fastapi import APIRouter, Depends, Query, Request
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
 from pydantic import ValidationError
 
-from api.dependencies import get_archive_offer_use_case
+from api.dependencies import get_archive_offer_use_case, get_load_offer_details_use_case
 from api.talentsoft import verify_talentsoft_signature
 from application.use_cases.archive_offer import ArchiveOfferUseCase
+from application.use_cases.load_offer_details import LoadOfferDetailsUseCase
 from presentation.dtos.talentsoft_webhook import (
     TalentsoftWebhookPayload,
     should_archive,
+    should_load_offer_details,
 )
 
 logger = logging.getLogger(__name__)
 
 public_router = APIRouter()
 
+_OK = {"status": "ok"}
+
 
 @public_router.get("/health")
 def health():
@@ -27,35 +31,61 @@ def health():
 async def talentsoft_webhook(
     request: Request,
     client_id: str = Query(...),
-    use_case: ArchiveOfferUseCase = Depends(get_archive_offer_use_case),
+    archive_use_case: ArchiveOfferUseCase | None = Depends(get_archive_offer_use_case),
+    load_offer_use_case: LoadOfferDetailsUseCase | None = Depends(
+        get_load_offer_details_use_case
+    ),
 ):
     body = await request.body()
     logger.debug(
         "Received TalentSoft webhook body",
         extra={"body": body.decode(), "client_id": client_id},
     )
     if not body:
-        return {"status": "ok"}
+        return _OK
+
+    payload = _parse_payload(body, client_id)
+    if payload is None:
+        return _OK
+
+    if should_archive(payload):
+        if archive_use_case is None:
+            raise HTTPException(status_code=500, detail="Web service not configured")
+        return await _handle_archive(payload, client_id, archive_use_case)
+
+    if should_load_offer_details(payload):
+        if load_offer_use_case is None:
+            raise HTTPException(
+                status_code=500, detail="Talentsoft client not configured"
+            )
+        return await _handle_load_offer_details(payload, client_id, load_offer_use_case)
+
+    logger.info(
+        "Unhandled event type %s for reference %s and status_id %s",
+        payload.event_type,
+        payload.reference,
+        payload.status_id,
+        extra={"client_id": client_id},
+    )
+    return _OK
+
 
+def _parse_payload(body: bytes, client_id: str) -> TalentsoftWebhookPayload | None:
     try:
-        payload = TalentsoftWebhookPayload.model_validate_json(body)
+        return TalentsoftWebhookPayload.model_validate_json(body)
     except ValidationError:
         logger.warning(
             "Unrecognised TalentSoft webhook payload, ignoring",
             extra={"body": body.decode(), "client_id": client_id},
         )
-        return {"status": "ok"}
-
-    if not should_archive(payload):
-        logger.info(
-            "Unhandled event type %s for reference %s and status_id %s",
-            payload.event_type,
-            payload.reference,
-            payload.status_id,
-            extra={"client_id": client_id},
-        )
-        return {"status": "ok"}
+        return None
+
 
+async def _handle_archive(
+    payload: TalentsoftWebhookPayload,
+    client_id: str,
+    use_case: ArchiveOfferUseCase,
+) -> dict:
     # `source_id` will not be `client_id` soon
     # https://github.com/betagouv/csplab/issues/573 is required
     await use_case.execute(reference=payload.reference, source_id=client_id)
@@ -65,4 +95,19 @@ async def talentsoft_webhook(
         payload.reference,
         extra={"client_id": client_id},
     )
-    return {"status": "ok"}
+    return _OK
+
+
+async def _handle_load_offer_details(
+    payload: TalentsoftWebhookPayload,
+    client_id: str,
+    use_case: LoadOfferDetailsUseCase,
+) -> dict:
+    await use_case.execute(reference=payload.reference)
+    logger.info(
+        "Handled event type %s for reference %s",
+        payload.event_type,
+        payload.reference,
+        extra={"client_id": client_id},
+    )
+    return _OK

src/ingestion/application/interfaces/__init__.py

@@ -0,0 +1,7 @@
+from typing import Protocol
+
+from infrastructure.external_gateways.dtos.talentsoft_dtos import TalentsoftDetailOffer
+
+
+class ITalentsoftFrontClient(Protocol):
+    async def get_detail(self, reference: str) -> TalentsoftDetailOffer: ...

src/ingestion/application/interfaces/talentsoft_client_interface.py

@@ -0,0 +1,10 @@
+from application.interfaces.talentsoft_client_interface import ITalentsoftFrontClient
+from infrastructure.external_gateways.dtos.talentsoft_dtos import TalentsoftDetailOffer
+
+
+class LoadOfferDetailsUseCase:
+    def __init__(self, talentsoft_client: ITalentsoftFrontClient) -> None:
+        self._talentsoft_client = talentsoft_client
+
+    async def execute(self, reference: str) -> TalentsoftDetailOffer:
+        return await self._talentsoft_client.get_detail(reference)