Merge remote-tracking branch 'origin/main' into feat/1170-merge-nvi-beheer-db

Author: jaytaph

app.conf.example

@@ -49,24 +49,14 @@ mtls_key=secrets/ssl/crypto_service_api.key
 verify_ca=secrets/ssl/crypto_service_api_ca.cert
 
 [telemetry]
-# Telemetry is enabled or not
-enabled = False
+# Push NVI usage metrics over OTLP or not
+metrics_enabled = False
 # Endpoint to GRPC server for telemetry
-endpoint = http://tracing:4317
+endpoint = http://otel-collector:4317
 # Service name to use
-service_name = Nationale Verwijsindex
-# Tracer name to use
-tracer_name = zmodules.service.national_referral_index
-
-[stats]
-# Statsd is enabled or not
-enabled = False
-# Host for statsd server
-host = 127.0.0.1
-# Port for statsd server
-port = 8125
-# Module name for statsd
-module_name = localisation
+service_name = nationale-verwijsindex
+# Interval in milliseconds between metric exports
+metrics_export_interval_ms = 15000
 
 [uvicorn]
 # If true, the api docs will be enabled
@@ -98,3 +88,9 @@ ssl_key_file = server.key
 # space separated list of expected audiences, one should be present in the authorization authorization header
 # usually the value is the domain of the running application
 expected_audiences = https://localhost:8501
+
+[pagination]
+# Number of entries returned per page when a search does not specify `_count`
+default_page_size = 50
+# Upper limit for `_count`, larger values are clamped to this value
+max_page_size = 100

app/application.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import os
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Iterator
 from contextlib import asynccontextmanager
 from pathlib import Path
 from typing import Any
@@ -13,21 +13,22 @@
 from gfmodules.logging.middleware import RequestContextMiddleware
 
 from app import container
-from app.auth import get_auth_ctx
+from app.auth import auth_log_context, get_auth_ctx
 from app.config import (
     _ENVIRONMENT_CONFIG_PATH_NAME,
     _PATH,
     get_config,
 )
 from app.errors.handlers import register_exceptions
+from app.logging.context import CLAIM_FIELDS
 from app.logging.events import Log
+from app.models.fhir.resources.localization_list.request import COUNT_PARAM
 from app.routers.default import router as default_router
 from app.routers.fhir.base import router as fhir_base_router
 from app.routers.fhir.localization_list import router as fhir_list_router
 from app.routers.health import router as health_router
 from app.routers.localize import router as localization_router
 from app.routers.registrations import router as registrations_router
-from app.stats import StatsdMiddleware
 
 logger = logging.getLogger(__name__)
 
@@ -83,7 +84,12 @@ def create_fastapi_app() -> FastAPI:
 
 def setup_logging() -> None:
     config = get_config()
-    gflog.configure(config=config.logging, loglevel=config.app.loglevel, catalogue=Log)
+    gflog.configure(
+        config=config.logging,
+        loglevel=config.app.loglevel,
+        catalogue=Log,
+        extra_context_fields=CLAIM_FIELDS,
+    )
 
 
 def _read_version() -> str:
@@ -123,6 +129,42 @@ def api_key_headers(document_gf_headers: bool) -> list[Any]:
     return [Security(APIKeyHeader(name=header, scheme_name=header, auto_error=False)) for header in headers]
 
 
+def _count_params(schema: dict[str, Any]) -> Iterator[dict[str, Any]]:
+    for operations in schema.get("paths", {}).values():
+        for operation in operations.values():
+            yield from (
+                param
+                for param in operation.get("parameters", [])
+                if param.get("in") == "query" and param.get("name") == COUNT_PARAM
+            )
+
+
+def _set_page_sizes(param: dict[str, Any], default_page_size: int, max_page_size: int) -> None:
+    param["description"] = (
+        f"Number of entries per page. Defaults to {default_page_size}, larger values are capped at {max_page_size}"
+    )
+    param["schema"]["default"] = default_page_size
+    for option in param["schema"].get("anyOf", [param["schema"]]):
+        if option.get("type") == "integer":
+            option["maximum"] = max_page_size
+
+
+def _document_page_sizes(app: FastAPI, default_page_size: int, max_page_size: int) -> None:
+    """
+    Shows the configured page sizes on the `_count` parameter in the OpenAPI schema. The maximum is not
+    validated on the parameter itself: FHIR treats `_count` as a hint, so a larger value is capped instead of rejected.
+    """
+    generate = app.openapi
+
+    def openapi() -> dict[str, Any]:
+        if app.openapi_schema is None:
+            for param in _count_params(generate()):
+                _set_page_sizes(param, default_page_size, max_page_size)
+        return app.openapi_schema  # type: ignore[return-value]
+
+    app.openapi = openapi  # type: ignore[method-assign]
+
+
 def setup_fastapi() -> FastAPI:
     config = get_config()
     fastapi = (
@@ -152,12 +194,10 @@ def setup_fastapi() -> FastAPI:
         fastapi.include_router(router)
 
     for router in routers:
-        fastapi.include_router(router, dependencies=[Depends(get_auth_ctx)])
+        fastapi.include_router(router, dependencies=[Depends(get_auth_ctx), Depends(auth_log_context)])
 
     register_exceptions(fastapi)
-
-    if config.stats.enabled:
-        fastapi.add_middleware(StatsdMiddleware, module_name=config.stats.module_name or "default")
+    _document_page_sizes(fastapi, config.pagination.default_page_size, config.pagination.max_page_size)
 
     fastapi.add_middleware(
         RequestContextMiddleware,

app/auth.py

@@ -1,10 +1,12 @@
 import logging
+from collections.abc import AsyncIterator
 from typing import Annotated
 
 from fastapi import Depends
 from starlette.requests import Request
 
 from app import dependencies
+from app.logging.context import bind_auth_claims
 from app.models.auth.context import AuthContext, AuthenticationClaims
 from app.models.auth.headers import AuthHeaders
 from app.models.ura import UraNumber
@@ -40,6 +42,7 @@ def get_auth_ctx(
         ura_number=UraNumber(validated_auth_headers.ura),
         source_id=validated_auth_headers.source_id,
         oin=validated_auth_headers.oin,
+        acting_organization_name=validated_auth_headers.acting_organization_name,
         organization_name=validated_auth_headers.organization_name,
     )
     ctx = AuthContext(
@@ -49,3 +52,8 @@ def get_auth_ctx(
     )
     request.state.auth = ctx
     return ctx
+
+
+async def auth_log_context(ctx: Annotated[AuthContext, Depends(get_auth_ctx)]) -> AsyncIterator[None]:
+    with bind_auth_claims(ctx):
+        yield

app/config.py

@@ -1,12 +1,14 @@
 import configparser
 import logging
 import os
+import socket
 from enum import Enum
-from typing import Any
+from typing import Any, Self
+from uuid import uuid4
 
 from gfmodules.logging import ConfigLogging as GFConfigLogging
 from gfmodules.logging.ini import split_comma_separated
-from pydantic import BaseModel, Field, SecretStr, ValidationError, field_validator
+from pydantic import BaseModel, Field, SecretStr, ValidationError, field_validator, model_validator
 
 logger = logging.getLogger(__name__)
 
@@ -67,18 +69,16 @@ class ConfigUvicorn(BaseModel):
     root_path: str = Field(default="")
 
 
+def process_instance_id() -> str:
+    return f"{socket.gethostname()}-{uuid4().hex[:8]}"
+
+
 class ConfigTelemetry(BaseModel):
-    enabled: bool = Field(default=False)
+    metrics_enabled: bool = Field(default=False)
     endpoint: str | None = None
     service_name: str | None = None
-    tracer_name: str | None = None
-
-
-class ConfigStats(BaseModel):
-    enabled: bool = Field(default=False)
-    host: str | None = None
-    port: int | None = None
-    module_name: str | None = None
+    instance_id: str = Field(default_factory=process_instance_id)
+    metrics_export_interval_ms: int = Field(default=15000, gt=0)
 
 
 class ConfigAuthorizationHeaders(BaseModel):
@@ -96,6 +96,17 @@ def validate_aud(cls, data: Any) -> list[str]:
         raise ValueError("Invalid input on `expected_audience`, please check config")
 
 
+class ConfigPagination(BaseModel):
+    default_page_size: int = Field(default=50, ge=1)
+    max_page_size: int = Field(default=100, ge=1)
+
+    @model_validator(mode="after")
+    def validate_page_sizes(self) -> Self:
+        if self.default_page_size > self.max_page_size:
+            raise ValueError("`default_page_size` cannot be larger than `max_page_size`")
+        return self
+
+
 class ConfigLogging(GFConfigLogging):
     _split_console_streams = field_validator("console_streams", mode="before")(split_comma_separated())
 
@@ -106,9 +117,9 @@ class Config(BaseModel):
     database: ConfigDatabase
     crypto_service_api: ConfigCryptoServiceApi
     telemetry: ConfigTelemetry
-    stats: ConfigStats
     uvicorn: ConfigUvicorn
     authorization_headers: ConfigAuthorizationHeaders
+    pagination: ConfigPagination = Field(default_factory=ConfigPagination)
 
 
 def read_ini_file(path: str) -> Any:

app/container.py

@@ -9,13 +9,16 @@
 )
 from app.db.db import Database
 from app.debug.crypto_service_api_client_mock import CryptoServiceApiClientMock
+from app.metrics import UsageMetrics
 from app.services.auth.header import AuthHeaderService
 from app.services.crypto_service_api_client import CryptoServiceApiClient
 from app.services.fhir.bundle import BundleService
 from app.services.fhir.localization_list import LocalizationListService
 from app.services.key_info import KeyInfoService, KeyInfoServiceMock
+from app.services.pseudonym_referral_service import PseudonymReferralService
 from app.services.pseudonym_resolver import PseudonymResolver
 from app.services.referral_service import ReferralService
+from app.telemetry import create_meter_provider
 from app.utils.load_capability_statement import (
     CapabilityStatement,
     load_capability_statement,
@@ -28,14 +31,16 @@ def container_config(binder: inject.Binder) -> None:
     config = get_config()
 
     binder.bind(Config, config)
+    meter_provider = create_meter_provider(config.telemetry)
 
     capability_statement = load_capability_statement()
     binder.bind(CapabilityStatement, capability_statement)
 
     db = Database(config_database=config.database)
     binder.bind(Database, db)
 
-    referral_service = ReferralService(database=db)
+    usage_metrics = UsageMetrics(meter_provider)
+    referral_service = ReferralService(database=db, usage_metrics=usage_metrics)
     binder.bind(ReferralService, referral_service)
 
     key_info_service = create_key_info_service(config.crypto_service_api, db)
@@ -51,11 +56,17 @@ def container_config(binder: inject.Binder) -> None:
         crypto_client=crypto_client,
         key_info_service=key_info_service,
     )
-    binder.bind(PseudonymResolver, pseudonym_resolver)
+    pseudonym_referral_service = PseudonymReferralService(
+        referral_service=referral_service,
+        pseudonym_resolver=pseudonym_resolver,
+    )
+    binder.bind(PseudonymReferralService, pseudonym_referral_service)
 
     localization_list_service = LocalizationListService(
         referral_service=referral_service,
-        pseudonym_resolver=pseudonym_resolver,
+        pseudonym_referral_service=pseudonym_referral_service,
+        default_page_size=config.pagination.default_page_size,
+        max_page_size=config.pagination.max_page_size,
     )
     binder.bind(LocalizationListService, localization_list_service)
 

app/db/models/referral.py

@@ -2,7 +2,7 @@
 from typing import TYPE_CHECKING
 from uuid import UUID, uuid4
 
-from sqlalchemy import TIMESTAMP, ForeignKey, String, UniqueConstraint
+from sqlalchemy import TIMESTAMP, ForeignKey, Index, String
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 from sqlalchemy.types import Uuid
 
@@ -15,12 +15,9 @@
 class ReferralEntity(Base):
     __tablename__ = "referrals"
     __table_args__ = (
-        UniqueConstraint(
-            "ura_number",
-            "pseudonym",
-            "source",
-            name="providers_unique_idx",
-        ),
+        Index("referrals_ura_pseudonym_source_key", "ura_number", "pseudonym", "source", unique=True),
+        Index("referrals_pseudonym_idx", "pseudonym"),
+        Index("referrals_ura_created_at_id_idx", "ura_number", "created_at", "id"),
     )
 
     id: Mapped[UUID] = mapped_column("id", Uuid, primary_key=True, default=uuid4)