Release/2025.06.3 [main]

Pipfile

@@ -8,10 +8,13 @@ aio-pika = "==9.5.*"
 aiofiles = "==24.1.0"
 aiohttp = "==3.12.7"
 alembic = "==1.16.1"
+asgi-correlation-id = "==4.3.4"
 asyncpg = "==0.30.0"
 azure-storage-blob = "==12.25.*"
 bcrypt = "==4.3.0"
 boto3 = "==1.38.28"
+bytecode = "==0.16.*"
+ddtrace = "==2.21.*"
 fastapi = "==0.115.*"
 fastapi-mail = "==1.2.9"
 firebase-admin = "==6.8.*"
@@ -24,20 +27,18 @@ pyjwt = "==2.10.1"
 pymongo = "==4.13.0"
 pyOpenSSL = "==25.1.0"
 python-multipart = "==0.0.20"
+python-slugify = "==8.0.4"
 redis = "==5.2.*"
 sentry-sdk = "~=2.13"
 sqlalchemy = { extras = ["asyncio"], version = "==1.4.53" }
 sqlalchemy-utils = "==0.41.2"
+structlog = "==25.4.0"
 taskiq = { extras = ["reload"], version = "==0.11.*" }
 taskiq-aio-pika = "==0.4.2"
 taskiq-fastapi = "==0.3.*"
 taskiq-redis = "==1.0.8"
 typer = "==0.16.0"
 uvicorn = { extras = ["standard"], version = "==0.34.*" }
-ddtrace = "==2.21.*"
-bytecode = "==0.16.*"
-structlog = "==25.4.0"
-asgi-correlation-id = "==4.3.4"
 
 [dev-packages]
 allure-pytest = "==2.14.*"
@@ -47,14 +48,15 @@ greenlet = "==3.2.2"
 ipdb = "==0.13.13"
 mypy = "==1.16.0"
 nest-asyncio = "==1.6.0"
+polyfactory = "==2.21.0"
 pre-commit = "==4.2.*"
 pudb = "==2025.1"
-polyfactory = "==2.21.0"
 pyld = "==2.0.4"
 pytest = "==8.4.0"
 pytest-asyncio = "==1.0.0"
 pytest-cov = "==6.1.1"
 pytest-env = "==1.1.5"
+pytest-httpx = "*"
 pytest-lazy-fixtures = "==1.*"
 pytest-mock = "==3.14.1"
 reproschema = "==0.6.2"
@@ -63,10 +65,9 @@ types-aiofiles = "==24.1.0.*"
 types-cachetools = "==6.0.0.20250525"
 types-python-dateutil = "==2.9.0.*"
 types-pytz = "==2025.2.0.*"
+types-pyyaml = "==6.0.12.20250516"
 types-requests = "==2.32.0.*"
 typing-extensions = "==4.12.2"
-pytest-httpx = "*"
-types-pyyaml = "==6.0.12.20250516"
 
 [requires]
 python_version = "3.13"

README.md

@@ -787,3 +787,52 @@ In case of Azure blob, specify your connection string into field `storage_secret
 Common Public Attribution License Version 1.0 (CPAL-1.0)
 
 Refer to [LICENSE.md](./LICENSE.MD)
+
+---
+
+## Command Line Interface (CLI)
+
+This project provides a powerful CLI for backend operations, available via `src/cli.py` and powered by [Typer](https://typer.tiangolo.com/). You can use it to manage migrations, patches, encryption, applet seeding, arbitrary server settings, and more.
+
+### Usage
+
+```bash
+python src/cli.py [COMMAND] [SUBCOMMAND] [OPTIONS]
+```
+
+### Available Top-Level Commands
+- `arbitrary` – Manage arbitrary server settings and data transfer
+- `patch` – Execute or list database/data patches
+- `encryption` – Encrypt, decrypt, or re-encrypt data
+- `applet` – Applet management and seeding
+- `applet-ema` – Export EMA schedules
+- `activities` – Commands for processing activities
+- `assessments` – Commands for processing assessments
+- `token` - Generate access token
+
+### Getting Help
+All commands and subcommands support `--help` for detailed usage, arguments, and options:
+
+```bash
+python src/cli.py [COMMAND] --help
+```
+
+### Example Commands
+- Run a patch:
+  ```bash
+  python src/cli.py patch exec M2-8568 -a <applet_id>
+  ```
+- Seed applet data from YAML:
+  ```bash
+  python src/cli.py applet seed /path/to/config.yaml
+  ```
+- Add arbitrary server settings:
+  ```bash
+  python src/cli.py arbitrary add <owner_email> --db-uri <uri> --storage-type <type> --storage-secret-key <key>
+  ```
+
+### More CLI Documentation
+Some commands (such as applet seeding) have detailed documentation in their respective subfolders, e.g.:
+- [`src/apps/applets/commands/applet/seed/v1/README.md`](src/apps/applets/commands/applet/seed/v1/README.md)
+
+Refer to these files for configuration schemas and advanced usage.

src/apps/answers/api.py

@@ -1,10 +1,14 @@
 import asyncio
 import base64
 import datetime
+import http
+import io
 import uuid
+import zipfile
 from typing import Annotated
 
 from fastapi import Body, Depends, Header, Query
+from fastapi import Response as FastAPIResponse
 from fastapi.responses import Response as FastApiResponse
 from pydantic import parse_obj_as
 
@@ -36,8 +40,13 @@
     PublicSummaryActivityFlow,
     ReviewsCount,
 )
-from apps.answers.domain.answers import MultiinformantAssessmentValidationResponse, PublicSubmissionsResponse
+from apps.answers.domain.answers import (
+    AnswerEHRFull,
+    MultiinformantAssessmentValidationResponse,
+    PublicSubmissionsResponse,
+)
 from apps.answers.filters import (
+    AnswerEHRExportFilters,
     AnswerExportFilters,
     AppletMultiinformantAssessmentParams,
     AppletSubmissionsFilter,
@@ -52,6 +61,8 @@
 from apps.applets.errors import InvalidVersionError, NotValidAppletHistory
 from apps.applets.service import AppletHistoryService, AppletService
 from apps.authentication.deps import get_current_user
+from apps.integrations.oneup_health.service.domain import EHRData
+from apps.integrations.oneup_health.service.ehr_storage import create_ehr_storage
 from apps.integrations.prolific.domain import ProlificUserInfo
 from apps.schedule.crud.user_device_events_history import UserDeviceEventsHistoryCRUD
 from apps.schedule.service.schedule_history import ScheduleHistoryService
@@ -116,7 +127,10 @@ async def create_answer(
         await service.create_report_from_answer(answer)
         if schema.allowed_ehr_ingest:
             await service.trigger_ehr_ingestion(
-                applet_id=answer.applet_id, submit_id=answer.submit_id, activity_id=schema.activity_id
+                target_subject_id=answer.target_subject_id,
+                applet_id=answer.applet_id,
+                submit_id=answer.submit_id,
+                activity_id=schema.activity_id,
             )
 
 
@@ -921,3 +935,46 @@ async def applet_submissions_list(
     return PublicSubmissionsResponse(
         submissions=submissions, submissions_count=submissions_count, participants_count=participants_count
     )
+
+
+async def applet_ehr_answers_export(
+    applet_id: uuid.UUID,
+    user: User = Depends(get_current_user),
+    session=Depends(get_session),
+    answer_session=Depends(get_answer_session),
+    query_params: QueryParams = Depends(parse_query_params(AnswerEHRExportFilters)),
+) -> FastAPIResponse:
+    await AppletService(session, user.id).exist_by_id(applet_id)
+    await CheckAccessService(session, user.id).check_answers_export_access(applet_id)
+
+    ehr_answers: list[AnswerEHRFull] = await AnswerService(session, user.id, answer_session).export_ehr_answers(
+        applet_id, query_params
+    )
+
+    if len(ehr_answers) == 0:
+        return FastAPIResponse(status_code=http.HTTPStatus.NO_CONTENT, content="No EHR answers found")
+
+    ehr_storage = await create_ehr_storage(session=session, applet_id=applet_id)
+    zip_buffer = io.BytesIO()
+    try:
+        with zipfile.ZipFile(zip_buffer, "w", compression=zipfile.ZIP_DEFLATED) as zip_file:
+            for ehr_answer in ehr_answers:
+                data = EHRData(
+                    target_subject_id=ehr_answer.target_subject_id,
+                    activity_id=ehr_answer.activity_id,
+                    submit_id=ehr_answer.submit_id,
+                    date=ehr_answer.date,
+                )
+                ehr_zip_buffer = io.BytesIO()
+                try:
+                    ehr_zip_filename = ehr_storage.download_ehr_zip(data, ehr_zip_buffer)
+
+                    zip_file.writestr(ehr_zip_filename, ehr_zip_buffer.getvalue())
+                finally:
+                    ehr_zip_buffer.close()
+
+        zip_buffer.seek(0)
+        headers = {"Content-Disposition": "attachment; filename=EHR.zip"}
+        return FastAPIResponse(zip_buffer.getvalue(), headers=headers, media_type="application/zip")
+    finally:
+        zip_buffer.close()

src/apps/answers/crud/answers.py

@@ -49,6 +49,14 @@ def filter_respondent_ids(self, field, value):
     to_date = FilterField(AnswerItemSchema.created_at, Comparisons.LESS_OR_EQUAL)
 
 
+class _AnswersEHRExportFilter(Filtering):
+    respondent_ids = FilterField(AnswerSchema.respondent_id, Comparisons.IN)
+    target_subject_ids = FilterField(AnswerSchema.target_subject_id, Comparisons.IN)
+    activity_ids = FilterField(AnswerEHRSchema.activity_id, Comparisons.IN)
+    from_date = FilterField(AnswerEHRSchema.created_at, Comparisons.GREAT_OR_EQUAL)
+    to_date = FilterField(AnswerEHRSchema.created_at, Comparisons.LESS_OR_EQUAL)
+
+
 class _AnswerListFilter(Filtering):
     respondent_ids = FilterField(AnswerItemSchema.respondent_id, method_name="filter_respondent_ids")
     target_subject_ids = FilterField(AnswerSchema.target_subject_id, Comparisons.IN)
@@ -1188,3 +1196,99 @@ async def upsert(self, schema: AnswerEHR) -> AnswerEHRSchema:
 
         result = await self._execute(stmt)
         return result.scalar_one_or_none()
+
+    async def get_by_applet_id(self, applet_id: uuid.UUID) -> list[AnswerEHRSchema]:
+        """
+        Get a list of AnswerEHRSchema records filtered by applet_id.
+
+        Since AnswerEHRSchema doesn't have an applet_id field directly, this method
+        joins with the AnswerSchema table to filter by applet_id.
+
+        Args:
+            applet_id: The UUID of the applet to filter by
+
+        Returns:
+            A list of AnswerEHRSchema objects associated with the given applet_id
+        """
+        query = (
+            select(self.schema_class)
+            .join(AnswerSchema, self.schema_class.submit_id == AnswerSchema.submit_id)
+            .where(AnswerSchema.applet_id == applet_id)
+        )
+
+        result = await self._execute(query)
+        return result.scalars().all()
+
+    async def get_ehr_answers_with_filters(
+        self, applet_id: uuid.UUID, activity_ids: list[uuid.UUID], submit_ids: list[uuid.UUID]
+    ) -> list[dict]:
+        query = (
+            select(
+                AnswerSchema.submit_id,
+                AnswerEHRSchema.activity_id,
+                AnswerEHRSchema.ehr_storage_uri,
+                AnswerEHRSchema.updated_at.label("date"),
+                AnswerSchema.respondent_id.label("user_id"),
+                AnswerEHRSchema.ehr_ingestion_status,
+            )
+            .join(AnswerSchema, self.schema_class.submit_id == AnswerSchema.submit_id)
+            .where(
+                AnswerSchema.applet_id == applet_id,
+                AnswerEHRSchema.activity_id.in_(activity_ids),
+                AnswerSchema.submit_id.in_(submit_ids),
+            )
+            .order_by(self.schema_class.created_at.desc())
+        )
+
+        result = await self._execute(query)
+        return result.mappings().all()
+
+    async def export_ehr_answers(self, applet_id: uuid.UUID, **filters) -> list[dict]:
+        """
+        Get a list of AnswerEHRSchema records filtered by applet_id for export purposes.
+
+        This method joins the AnswerEHRSchema with the AnswerSchema table to filter by applet_id.
+
+        Args:
+            applet_id: The UUID of the applet to filter by
+
+        Returns:
+            A list of AnswerEHRSchema objects associated with the given applet_id
+        """
+
+        filter_clauses = []
+        if filters:
+            filter_clauses = _AnswersEHRExportFilter().get_clauses(**filters)
+
+            if flow_ids := filters.get("flow_ids"):
+                submit_ids_sub_query = (
+                    select(AnswerSchema.submit_id)
+                    .where(AnswerSchema.applet_id == applet_id)
+                    .where(
+                        AnswerSchema.id_from_history_id(AnswerSchema.flow_history_id).in_(
+                            [str(flow_id) for flow_id in flow_ids]
+                        )
+                    )
+                    .distinct()
+                )
+                filter_clauses.append(AnswerEHRSchema.submit_id.in_(submit_ids_sub_query))
+
+        query = (
+            select(
+                AnswerSchema.submit_id,
+                AnswerEHRSchema.activity_id,
+                AnswerEHRSchema.ehr_storage_uri,
+                AnswerEHRSchema.updated_at.label("date"),
+                AnswerSchema.target_subject_id,
+                AnswerEHRSchema.ehr_ingestion_status,
+            )
+            .join(AnswerSchema, self.schema_class.submit_id == AnswerSchema.submit_id)
+            .where(AnswerSchema.applet_id == applet_id)
+            .where(AnswerEHRSchema.ehr_ingestion_status == EHRIngestionStatus.COMPLETED)
+            .where(AnswerEHRSchema.ehr_storage_uri.is_not(None))
+            .where(*filter_clauses)
+            .order_by(self.schema_class.created_at.desc())
+        )
+
+        result = await self._execute(query)
+        return result.mappings().all()

src/apps/answers/domain/answers.py

@@ -566,6 +566,7 @@ class UserAnswerDataBase(BaseModel):
     created_at: datetime.datetime
     migrated_data: dict | None = None
     client: ClientMeta | None = None
+    ehr_data_file: str | None = None
 
 
 class RespondentAnswerData(UserAnswerDataBase, InternalModel):
@@ -747,3 +748,8 @@ class AnswerEHR(InternalModel):
     ehr_ingestion_status: EHRIngestionStatus
     activity_id: uuid.UUID
     ehr_storage_uri: str | None
+
+
+class AnswerEHRFull(AnswerEHR):
+    target_subject_id: uuid.UUID
+    date: datetime.datetime