Update DQ email PDF rendering and API integration

Author: reanbrenda

api/data_ingestion/internal/email.py

@@ -1,3 +1,4 @@
+import base64
 from typing import Any
 
 import requests
@@ -182,7 +183,7 @@ def send_dq_report_email_with_pdf(body: EmailRenderRequest[DqReportRenderRequest
     html_content = email_data.get("html")
     text_content = email_data.get("text")
 
-    # Generate PDF
+    # Generate PDF (renderer returns binary to avoid large JSON truncation)
     pdf_res = requests.post(
         f"{settings.EMAIL_RENDERER_SERVICE_URL}/email/dq-report-pdf",
         headers={
@@ -198,11 +199,14 @@ def send_dq_report_email_with_pdf(body: EmailRenderRequest[DqReportRenderRequest
         except JSONDecodeError:
             raise HTTPError(pdf_res.text) from None
 
-    pdf_data = pdf_res.json()
-    pdf_base64 = pdf_data.get("pdf")
-    pdf_filename = pdf_data.get(
-        "filename", f"data-quality-report-{body.props.country}.pdf"
-    )
+    pdf_bytes = pdf_res.content
+    pdf_base64 = base64.b64encode(pdf_bytes).decode("ascii")
+    disp = pdf_res.headers.get("Content-Disposition") or ""
+    pdf_filename = f"data-quality-report-{body.props.country}.pdf"
+    if "filename=" in disp:
+        part = disp.split("filename=", 1)[1].strip().strip('"')
+        if part:
+            pdf_filename = part
 
     # Use base64 string directly as required by Mailjet v3 send API
     attachment = {
@@ -263,7 +267,15 @@ async def generate_dq_report_pdf(body: EmailRenderRequest[DqReportRenderRequest]
         except JSONDecodeError:
             raise HTTPError(res.text) from None
     logger.info(f"PDF generation response: {res.status_code}")
-    return res.json()
+    pdf_bytes = res.content
+    pdf_b64 = base64.b64encode(pdf_bytes).decode("ascii")
+    disp = res.headers.get("Content-Disposition") or ""
+    filename = f"data-quality-report-{body.props.country}-{body.props.uploadId}.pdf"
+    if "filename=" in disp:
+        part = disp.split("filename=", 1)[1].strip().strip('"')
+        if part:
+            filename = part
+    return {"pdf": pdf_b64, "filename": filename}
 
 
 async def generate_dq_report_pdf_from_payload(payload: dict) -> dict:
@@ -283,7 +295,15 @@ async def generate_dq_report_pdf_from_payload(payload: dict) -> dict:
         except JSONDecodeError:
             raise HTTPError(res.text) from None
     logger.info(f"PDF generation response: {res.status_code}")
-    return res.json()
+    pdf_bytes = res.content
+    pdf_b64 = base64.b64encode(pdf_bytes).decode("ascii")
+    disp = res.headers.get("Content-Disposition") or ""
+    filename = "data-quality-report.pdf"
+    if "filename=" in disp:
+        part = disp.split("filename=", 1)[1].strip().strip('"')
+        if part:
+            filename = part
+    return {"pdf": pdf_b64, "filename": filename}
 
 
 def send_master_data_release_notification(

email/package-lock.json

@@ -15,7 +15,7 @@
     "@hono/zod-validator": "^0.2.2",
     "@react-email/components": "^0.0.24",
     "@react-email/render": "^1.0.1",
-    "@react-pdf/renderer": "^3.4.4",
+    "@react-pdf/renderer": "^3.4.5",
     "@sentry/node": "^5.5.0",
     "@sentry/profiling-node": "^8.37.1",
     "hono": "^4.5.9",

email/package.json

@@ -251,7 +251,7 @@ const DataQualityReportPDF: React.FC<PDFReportProps> = ({
             <Text style={styles.sectionTitle}>School ID Checks</Text>
             <MetricRow label="Duplicate School IDs" value={duplicateSchoolIds} />
             <MetricRow label="Missing School IDs" value={missingSchoolIds} />
-            <MetricRow label="Missing School Names D" value={missingSchoolNames} />
+            <MetricRow label="Missing School Names" value={missingSchoolNames} />
             <CommentBox>{''}</CommentBox>
           </View>
         </View>

email/src/emails/dq-report-pdf.tsx

@@ -112,13 +112,15 @@ app.post(
     };
 
     const pdfBuffer = await generateDataQualityReportPDF(pdfData);
-    
-    // Return PDF as base64 for easy attachment
-    const pdfBase64 = pdfBuffer.toString('base64');
-    
-    return ctx.json({ 
-      pdf: pdfBase64,
-      filename: `data-quality-report-${json.country}-${json.uploadId}.pdf`
+    const filename = `data-quality-report-${json.country}-${json.uploadId}.pdf`;
+
+    // Return PDF as binary to avoid huge JSON/base64 response truncation
+    return new Response(pdfBuffer, {
+      status: 200,
+      headers: {
+        "Content-Type": "application/pdf",
+        "Content-Disposition": `attachment; filename="${filename}"`,
+      },
     });
   },
 );

email/src/index.tsx

@@ -7,8 +7,8 @@ const Check = z.object({
   count_overall: z.number(),
   count_passed: z.number(),
   description: z.string(),
-  percent_failed: z.number(),
-  percent_passed: z.number(),
+  percent_failed: z.number().nullable(),
+  percent_passed: z.number().nullable(),
   dq_remarks: z.string().optional(),
 });