get the recon data too

Author: pld

odk-central-sync/README.md

@@ -36,3 +36,28 @@ Each run writes the following into `--output-dir`:
 - `<center_id>/results.zip` — raw ODK Central export per center (cached; reused on reruns)
 - `media/` — extracted images from all centers, filenames prefixed by `center_id`
 - `results_export_p<project_id>_<timestamp>.zip` — upload bundle containing `candidate_results.csv` and `media/`. This is the file to upload to the results system for integration with other results. Disable with `--bundle=false`.
+
+### `candidate_results.csv` columns
+
+One row per candidate per submission. Columns:
+
+- Candidate fields from the ODK `candidate_results` repeat:
+  `pos`, `candidate_id`, `candidate_order`, `candidate_name`,
+  `candidate_result_round1`, `candidate_result_round2` (flattened from the
+  ODK `candidate_result_r2-candidate_result_round2` path),
+  `candidate_result_r2-result_note`, `PARENT_KEY`, `KEY`.
+- Provenance: `xml_form_id`, `center_id`.
+- Submission fields joined on `PARENT_KEY` → `meta-instanceID`:
+  `station_number`, `staff_user_name`, `ballot_number`, `race_type`.
+- **`barcode`** — the PVP scanned barcode (renamed from the ODK
+  `intro-barcode` field). String type to preserve leading zeros.
+- **Reconciliation fields** — both rounds (r1 and r2) captured by the PVP
+  device, five fields each:
+  `reconciliation_r1-number_ballots_received_r1`,
+  `reconciliation_r1-number_voter_cards_r1`,
+  `reconciliation_r1-number_valid_ballots_r1`,
+  `reconciliation_r1-number_invalid_ballots_r1`,
+  `reconciliation_r1-number_ballots_inside_box_r1`,
+  and the equivalent `reconciliation_r2-*` columns.
+- Image filenames (prefixed with `center_id`): `clerk_signature`,
+  `forms_picture_1st_page`, `forms_picture_2nd_page`.

odk-central-sync/src/download_results_forms.py

@@ -83,6 +83,21 @@ def export_center_candidate_results(
         "forms_picture_1st_page",
         "forms_picture_2nd_page",
     ]
+    # PVP reconciliation fields. PVP devices collect two passes (r1, r2) of the
+    # reconciliation data per submission. Downstream tally-ho uses r2 but we
+    # export both so the raw data is preserved for review.
+    RECON_COLUMNS = [
+        "reconciliation_r1-number_ballots_received_r1",
+        "reconciliation_r1-number_voter_cards_r1",
+        "reconciliation_r1-number_valid_ballots_r1",
+        "reconciliation_r1-number_invalid_ballots_r1",
+        "reconciliation_r1-number_ballots_inside_box_r1",
+        "reconciliation_r2-number_ballots_received_r2",
+        "reconciliation_r2-number_voter_cards_r2",
+        "reconciliation_r2-number_valid_ballots_r2",
+        "reconciliation_r2-number_invalid_ballots_r2",
+        "reconciliation_r2-number_ballots_inside_box_r2",
+    ]
 
     candidate_results: list[pd.DataFrame] = []
     media_dir = output_dir / "media"
@@ -114,6 +129,15 @@ def export_center_candidate_results(
                 candidates_path = Path(f"{xml_form_id}-candidate_results.csv")
                 console.log(f"Extracting {candidates_path}")
                 df = pd.read_csv(archive.open(str(candidates_path)))
+                # PVP stores the second round of candidate votes under a
+                # hyphenated group path; flatten it to candidate_result_round2
+                # to match the round1 column naming.
+                df = df.rename(
+                    columns={
+                        "candidate_result_r2-candidate_result_round2":
+                            "candidate_result_round2",
+                    }
+                )
                 df["xml_form_id"] = xml_form_id
                 df["center_id"] = center_id
 
@@ -142,21 +166,32 @@ def export_center_candidate_results(
                             if pd.notna(v) else v
                         )
 
+                required_cols = [
+                    "meta-instanceID",
+                    "station_number",
+                    "staff_user_name",
+                    "ballot_number",
+                    "race_type",
+                ]
+                optional_cols = (
+                    ["intro-barcode"] + RECON_COLUMNS + IMAGE_COLUMNS
+                )
+                selected_cols = required_cols + [
+                    c for c in optional_cols if c in submissions_df.columns
+                ]
                 df = df.merge(
-                    submissions_df[
-                        [
-                            "meta-instanceID",
-                            "station_number",
-                            "staff_user_name",
-                            "ballot_number",
-                            "race_type",
-                        ]
-                        + [c for c in IMAGE_COLUMNS if c in submissions_df.columns]
-                    ],
+                    submissions_df[selected_cols],
                     left_on="PARENT_KEY",
                     right_on="meta-instanceID",
                     how="left",
                 ).drop(columns=["meta-instanceID"])
+                # Rename the PVP barcode column to a plain "barcode" for
+                # downstream consumers (tally-ho upload matches on barcode).
+                # Coerce to string so numeric-looking barcodes keep leading
+                # zeros and match ResultForm.barcode (CharField).
+                if "intro-barcode" in df.columns:
+                    df = df.rename(columns={"intro-barcode": "barcode"})
+                    df["barcode"] = df["barcode"].astype("string")
                 candidate_results.append(df)
 
             progress.advance(task)

odk-central-sync/tests/test_download_results_forms.py

@@ -19,14 +19,32 @@ def make_test_zip(center_id, include_media=True):
     form_id = f"results_{center_id}"
 
     candidates_csv = (
-        "pos,candidate_id,candidate_name,candidate_result_round1,PARENT_KEY,KEY\n"
-        f"1,101,Alice,25,uuid:abc-{center_id},uuid:abc-{center_id}/candidate_results[1]\n"
-        f"2,102,Bob,30,uuid:abc-{center_id},uuid:abc-{center_id}/candidate_results[2]\n"
+        "pos,candidate_id,candidate_name,"
+        "candidate_result_round1,candidate_result_r2-candidate_result_round2,"
+        "PARENT_KEY,KEY\n"
+        f"1,101,Alice,25,26,uuid:abc-{center_id},"
+        f"uuid:abc-{center_id}/candidate_results[1]\n"
+        f"2,102,Bob,30,31,uuid:abc-{center_id},"
+        f"uuid:abc-{center_id}/candidate_results[2]\n"
     )
     submissions_csv = (
         "meta-instanceID,station_number,staff_user_name,ballot_number,race_type,"
+        "intro-barcode,"
+        "reconciliation_r1-number_ballots_received_r1,"
+        "reconciliation_r1-number_voter_cards_r1,"
+        "reconciliation_r1-number_valid_ballots_r1,"
+        "reconciliation_r1-number_invalid_ballots_r1,"
+        "reconciliation_r1-number_ballots_inside_box_r1,"
+        "reconciliation_r2-number_ballots_received_r2,"
+        "reconciliation_r2-number_voter_cards_r2,"
+        "reconciliation_r2-number_valid_ballots_r2,"
+        "reconciliation_r2-number_invalid_ballots_r2,"
+        "reconciliation_r2-number_ballots_inside_box_r2,"
         "clerk_signature,forms_picture_1st_page,forms_picture_2nd_page\n"
         f"uuid:abc-{center_id},3,tester,1313,Individual,"
+        f"{center_id}003001,"
+        "200,150,140,10,150,"
+        "204,149,139,10,149,"
         f"sig_{center_id}.jpg,page1_{center_id}.jpg,\n"
     )
 
@@ -197,6 +215,102 @@ def test_image_columns_have_prefixed_filenames(self, output_dir):
         assert df["clerk_signature"].iloc[0] == f"{center_id}_sig_{center_id}.jpg"
         assert df["forms_picture_1st_page"].iloc[0] == f"{center_id}_page1_{center_id}.jpg"
 
+    def test_extracts_barcode(self, output_dir):
+        center_id = 100
+        zip_bytes = make_test_zip(center_id)
+
+        mock_response = MagicMock()
+        mock_response.content = zip_bytes
+        mock_response.raise_for_status = MagicMock()
+
+        mock_client = MagicMock()
+        mock_client.get.return_value = mock_response
+
+        df = export_center_candidate_results(
+            client=mock_client,
+            project_id=1,
+            center_ids=[center_id],
+            output_dir=output_dir,
+        )
+
+        assert "barcode" in df.columns
+        assert list(df["barcode"]) == [f"{center_id}003001"] * 2
+        # Original intro-barcode should have been renamed
+        assert "intro-barcode" not in df.columns
+
+    def test_extracts_round2_candidate_votes(self, output_dir):
+        center_id = 100
+        zip_bytes = make_test_zip(center_id)
+
+        mock_response = MagicMock()
+        mock_response.content = zip_bytes
+        mock_response.raise_for_status = MagicMock()
+
+        mock_client = MagicMock()
+        mock_client.get.return_value = mock_response
+
+        df = export_center_candidate_results(
+            client=mock_client,
+            project_id=1,
+            center_ids=[center_id],
+            output_dir=output_dir,
+        )
+
+        assert "candidate_result_round2" in df.columns
+        assert list(df["candidate_result_round2"]) == [26, 31]
+        # Original hyphenated column should have been renamed away
+        assert (
+            "candidate_result_r2-candidate_result_round2" not in df.columns
+        )
+        # Round 1 should still be present unchanged
+        assert "candidate_result_round1" in df.columns
+        assert list(df["candidate_result_round1"]) == [25, 30]
+
+    def test_extracts_reconciliation_fields(self, output_dir):
+        center_id = 100
+        zip_bytes = make_test_zip(center_id)
+
+        mock_response = MagicMock()
+        mock_response.content = zip_bytes
+        mock_response.raise_for_status = MagicMock()
+
+        mock_client = MagicMock()
+        mock_client.get.return_value = mock_response
+
+        df = export_center_candidate_results(
+            client=mock_client,
+            project_id=1,
+            center_ids=[center_id],
+            output_dir=output_dir,
+        )
+
+        r1_cols = [
+            "reconciliation_r1-number_ballots_received_r1",
+            "reconciliation_r1-number_voter_cards_r1",
+            "reconciliation_r1-number_valid_ballots_r1",
+            "reconciliation_r1-number_invalid_ballots_r1",
+            "reconciliation_r1-number_ballots_inside_box_r1",
+        ]
+        r2_cols = [
+            "reconciliation_r2-number_ballots_received_r2",
+            "reconciliation_r2-number_voter_cards_r2",
+            "reconciliation_r2-number_valid_ballots_r2",
+            "reconciliation_r2-number_invalid_ballots_r2",
+            "reconciliation_r2-number_ballots_inside_box_r2",
+        ]
+        for col in r1_cols + r2_cols:
+            assert col in df.columns, f"missing reconciliation column: {col}"
+
+        # Spot-check the values round-trip correctly
+        assert (
+            df["reconciliation_r1-number_ballots_received_r1"].iloc[0] == 200
+        )
+        assert df["reconciliation_r1-number_valid_ballots_r1"].iloc[0] == 140
+        assert (
+            df["reconciliation_r2-number_ballots_received_r2"].iloc[0] == 204
+        )
+        assert df["reconciliation_r2-number_valid_ballots_r2"].iloc[0] == 139
+
     def test_skips_existing_valid_zip(self, output_dir):
         center_id = 100
         zip_bytes = make_test_zip(center_id)