Merge branch 'main' into bob/refactor-csv-bulk-upload-pt-1

Author: fzhao99

refiner/app/api/v1/configurations/exports.py

@@ -2,9 +2,11 @@
 from datetime import UTC, datetime
 from io import StringIO
 from logging import Logger
+from typing import Literal
 from uuid import UUID
 
 from fastapi import APIRouter, Depends, HTTPException, Response, status
+from fastapi.responses import StreamingResponse
 
 from app.api.auth.middleware import get_logged_in_user
 from app.db.conditions.db import (
@@ -13,10 +15,17 @@
 )
 from app.db.conditions.model import DbCondition
 from app.db.configurations.db import get_configuration_by_id_db
-from app.db.configurations.model import DbConfiguration
+from app.db.configurations.model import (
+    DbConfiguration,
+    DbConfigurationSectionProcessing,
+    DbNarrativeAction,
+    DbSectionAction,
+)
 from app.db.pool import AsyncDatabaseConnection, get_db
 from app.db.users.model import DbUser
 from app.services.code_systems import get_all_code_systems_by_key
+from app.services.ecr.policy import NARRATIVE_ONLY_SECTIONS
+from app.services.file_io import ZipFileItem, ZipFilePackage
 from app.services.logger import get_logger
 
 router = APIRouter(prefix="/{configuration_id}/export")
@@ -52,25 +61,122 @@ async def get_configuration_export(
         included_conditions=config.included_conditions, db=db
     )
 
-    csv_bytes = await _build_config_csv(
+    codes_csv_content = await _build_config_csv(
         config=config, conditions=included_conditions, logger=logger, db=db
     )
+    sections_csv_content = _build_sections_csv(sections=config.section_processing)
 
-    filename = _build_export_filename(config_name=config.name)
+    timestamp = _get_timestamp()
 
-    return Response(
-        content=csv_bytes,
-        media_type="text/csv; charset=utf-8",
-        headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+    zip_package = ZipFilePackage(
+        name=_build_export_filename(
+            filename="Configuration_Export",
+            extension="zip",
+            config_name=config.name,
+            config_version=config.version,
+            timestamp=timestamp,
+        )
+    )
+
+    zip_package.add(
+        ZipFileItem(
+            file_name=_build_export_filename(
+                filename="Code_Export",
+                extension="csv",
+                config_name=config.name,
+                config_version=config.version,
+                timestamp=timestamp,
+            ),
+            file_content=codes_csv_content,
+        )
     )
 
+    zip_package.add(
+        ZipFileItem(
+            file_name=_build_export_filename(
+                filename="Section_Export",
+                extension="csv",
+                config_name=config.name,
+                config_version=config.version,
+                timestamp=timestamp,
+            ),
+            file_content=sections_csv_content,
+        )
+    )
+
+    return StreamingResponse(
+        content=zip_package.iter_chunks(),
+        media_type="application/zip",
+        headers={
+            "Content-Disposition": f'attachment; filename="{zip_package.get_name()}"'
+        },
+    )
+
+
+def _build_sections_csv(
+    sections: list[DbConfigurationSectionProcessing],
+) -> str:
+    """Build a CSV summarizing configuration section information."""
+    with StringIO() as csv_text:
+        writer = csv.writer(csv_text)
+        writer.writerow(
+            ["Section Name", "LOINC", "Include", "Coded Data", "Narrative Data"]
+        )
+
+        for section in sorted(sections, key=lambda r: r.name.lower()):
+            writer.writerow(
+                [
+                    section.name,
+                    section.code,
+                    "Yes" if section.include else "No",
+                    _get_coded_data_value(
+                        loinc=section.code,
+                        action=section.action,
+                        included=section.include,
+                    ),
+                    _get_narrative_data_value(
+                        narrative=section.narrative, included=section.include
+                    ),
+                ]
+            )
+        return csv_text.getvalue()
+
+
+def _get_coded_data_value(loinc: str, action: DbSectionAction, included: bool) -> str:
+    if not included:
+        return "N/A"
+
+    if loinc in NARRATIVE_ONLY_SECTIONS:
+        return "N/A"
+
+    if action == "retain":
+        return "Keep original"
+
+    if action == "refine":
+        return "Refine"
+
+    return "N/A"
+
+
+def _get_narrative_data_value(narrative: DbNarrativeAction, included: bool) -> str:
+    if not included:
+        return "N/A"
+
+    if narrative == "retain":
+        return "Keep original"
+
+    if narrative == "remove":
+        return "Exclude"
+
+    return "Reconstruct"
+
 
 async def _build_config_csv(
     config: DbConfiguration,
     conditions: list[DbCondition],
     logger: Logger,
     db: AsyncDatabaseConnection,
-) -> bytes:
+) -> str:
     """Build the CSV export content for a configuration."""
 
     code_systems = await get_all_code_systems_by_key(db=db)
@@ -113,11 +219,22 @@ async def _build_config_csv(
                 ]
             )
 
-        return csv_text.getvalue().encode("utf-8")
+        return csv_text.getvalue()
+
+
+def _get_timestamp() -> str:
+    now = datetime.now(UTC)
+    timestamp = now.strftime("%m%d%y_%H_%M_%S")
+    return timestamp
 
 
-def _build_export_filename(config_name: str) -> str:
+def _build_export_filename(
+    filename: str,
+    extension: Literal["csv", "zip"],
+    config_name: str,
+    config_version: int,
+    timestamp: str,
+) -> str:
     """Build a timestamped filename for a configuration export."""
-    safe_name = config_name.replace(" ", "_")
-    timestamp = datetime.now(UTC).strftime("%m%d%y_%H_%M_%S")
-    return f"{safe_name}_Code_Export_{timestamp}.csv"
+    condition_grouper = config_name.replace(" ", "_")
+    return f"{condition_grouper}_v{config_version}_{filename}_{timestamp}.{extension}"

refiner/app/services/file_io.py

@@ -1,8 +1,9 @@
 import io
 import time
+from collections.abc import Iterator
 from dataclasses import dataclass
 from io import BytesIO
-from zipfile import BadZipFile, ZipFile, ZipInfo
+from zipfile import ZIP_DEFLATED, BadZipFile, ZipFile, ZipInfo
 
 from chardet import detect
 from lxml import etree
@@ -69,6 +70,16 @@ def get_items(self) -> list[ZipFileItem]:
         """
         return list(self._packaged_items)
 
+    def iter_chunks(self) -> Iterator[bytes]:
+        """
+        Yields the zip file contents as chunks. Useful for usage with a StreamingResponse.
+        """
+        buffer = BytesIO()
+        with ZipFile(buffer, "w", ZIP_DEFLATED) as zf:
+            for item in self._packaged_items:
+                zf.writestr(item.file_name, item.file_content)
+        yield buffer.getvalue()
+
 
 def parse_xml(xml_content: str | bytes) -> _Element:
     """