stricter style, zip output, upgrade

Author: pld

odk-central-sync/README.md

@@ -26,3 +26,13 @@ uv run download-results --project-id=14
 - `--project-id` (required) — ODK Central project ID
 - `--output-dir` — Directory for output files (default: `results-output`)
 - `--config` — Path to pyODK config file (default: `.pyodk_config.toml`)
+- `--bundle` — Create a timestamped upload bundle in the output directory (default: `true`). Pass `--bundle=false` to skip.
+
+## Output
+
+Each run writes the following into `--output-dir`:
+
+- `candidate_results.csv` — combined candidate results across all centers
+- `<center_id>/results.zip` — raw ODK Central export per center (cached; reused on reruns)
+- `media/` — extracted images from all centers, filenames prefixed by `center_id`
+- `results_export_p<project_id>_<timestamp>.zip` — upload bundle containing `candidate_results.csv` and `media/`. This is the file to upload to the results system for integration with other results. Disable with `--bundle=false`.

odk-central-sync/pyproject.toml

@@ -24,4 +24,29 @@ packages = ["src"]
 [dependency-groups]
 dev = [
     "pytest>=9.0.2",
+    "ruff>=0.8",
 ]
+
+[tool.ruff]
+line-length = 100
+target-version = "py313"
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # pyflakes
+    "I",   # isort
+    "B",   # flake8-bugbear
+    "UP",  # pyupgrade
+    "SIM", # flake8-simplify
+    "TID", # tidy imports
+]
+
+[tool.ruff.lint.isort]
+force-single-line = true
+force-sort-within-sections = true
+lines-after-imports = 2
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "parents"

odk-central-sync/src/download_results_forms.py

@@ -1,16 +1,22 @@
-import zipfile
+from datetime import datetime
+import logging
 from pathlib import Path
+import zipfile
 
 import click
 import pandas as pd
 from pyodk.client import Client
 from pyodk.errors import PyODKError
-from requests.exceptions import ConnectionError, Timeout
+from requests.exceptions import ConnectionError
+from requests.exceptions import Timeout
 from rich.console import Console
 from rich.progress import Progress
-from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type, before_sleep_log
+from tenacity import before_sleep_log
+from tenacity import retry
+from tenacity import retry_if_exception_type
+from tenacity import stop_after_attempt
+from tenacity import wait_exponential
 
-import logging
 
 log = logging.getLogger(__name__)
 
@@ -24,7 +30,12 @@
     "  1) Create a pyODK config file (default: .pyodk_config.toml).\n"
     "     https://getodk.github.io/pyodk/#configure\n\n"
     "Example:\n"
-    "  uv run download-results --project-id=5 11034 11035 11036"
+    "  uv run download-results --project-id=5 11034 11035 11036\n\n"
+    "Output:\n"
+    "  - candidate_results.csv: combined results across all centers\n"
+    "  - media/: extracted images, filenames prefixed by center_id\n"
+    "  - results_export_p<project_id>_<timestamp>.zip: upload bundle\n"
+    "    (CSV + media/). Disable with --bundle=false."
 )
 
 
@@ -153,6 +164,32 @@ def export_center_candidate_results(
     return pd.concat(candidate_results, ignore_index=True)
 
 
+def create_upload_bundle(
+    output_dir: Path,
+    csv_path: Path,
+    media_dir: Path,
+    project_id: int,
+) -> Path:
+    """
+    Package the candidate results CSV and media files into a timestamped ZIP
+    for upload to the results system.
+
+    The bundle is written to output_dir as results_export_p<project_id>_<timestamp>.zip
+    with no compression (ZIP_STORED) since media is already compressed.
+    """
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    bundle_path = output_dir / f"results_export_p{project_id}_{timestamp}.zip"
+
+    with zipfile.ZipFile(bundle_path, "w", zipfile.ZIP_STORED) as zf:
+        zf.write(csv_path, arcname=csv_path.name)
+        if media_dir.exists():
+            for media_file in sorted(media_dir.iterdir()):
+                if media_file.is_file():
+                    zf.write(media_file, arcname=f"media/{media_file.name}")
+
+    return bundle_path
+
+
 @click.command(help=CLI_HELP, epilog=CLI_EPILOG)
 @click.option("--project-id", type=int, required=True, help="ODK Central project ID")
 @click.argument("center-ids", type=click.INT, nargs=-1)
@@ -170,7 +207,23 @@ def export_center_candidate_results(
     show_default=True,
     help="Path to pyODK config file. See https://getodk.github.io/pyodk/#configure",
 )
-def main(project_id: int, center_ids: tuple[int, ...], output_dir: Path, config: Path):
+@click.option(
+    "--bundle",
+    type=bool,
+    default=True,
+    show_default=True,
+    help=(
+        "Create a timestamped ZIP bundle (candidate_results.csv + media/) in the "
+        "output directory for upload to the results system. Pass --bundle=false to disable."
+    ),
+)
+def main(
+    project_id: int,
+    center_ids: tuple[int, ...],
+    output_dir: Path,
+    config: Path,
+    bundle: bool,
+):
     output_dir.mkdir(parents=True, exist_ok=True)
     output_csv = output_dir / "candidate_results.csv"
 
@@ -197,6 +250,15 @@ def main(project_id: int, center_ids: tuple[int, ...], output_dir: Path, config:
     results_df.to_csv(output_csv, index=False)
     console.log(f"Saved {len(results_df)} rows to {output_csv}")
 
+    if bundle:
+        bundle_path = create_upload_bundle(
+            output_dir=output_dir,
+            csv_path=output_csv,
+            media_dir=output_dir / "media",
+            project_id=project_id,
+        )
+        console.log(f"Created upload bundle: {bundle_path}")
+
 
 if __name__ == "__main__":
     main()

odk-central-sync/tests/test_download_results_forms.py

@@ -1,17 +1,17 @@
 import io
+import re
+from unittest.mock import MagicMock
+from unittest.mock import patch
 import zipfile
-from unittest.mock import MagicMock, patch
 
-import pytest
 from click.testing import CliRunner
-
 from pyodk.errors import PyODKError
+import pytest
 
-from src.download_results_forms import (
-    export_center_candidate_results,
-    export_form_submissions,
-    main,
-)
+from src.download_results_forms import create_upload_bundle
+from src.download_results_forms import export_center_candidate_results
+from src.download_results_forms import export_form_submissions
+from src.download_results_forms import main
 
 
 def make_test_zip(center_id, include_media=True):
@@ -220,6 +220,89 @@ def test_skips_existing_valid_zip(self, output_dir):
         assert len(df) == 2
 
 
+class TestCreateUploadBundle:
+    def test_bundle_contains_csv_and_media(self, tmp_path):
+        output_dir = tmp_path / "output"
+        output_dir.mkdir()
+        csv_path = output_dir / "candidate_results.csv"
+        csv_path.write_text("a,b\n1,2\n")
+        media_dir = output_dir / "media"
+        media_dir.mkdir()
+        (media_dir / "100_sig.jpg").write_bytes(b"img1")
+        (media_dir / "100_page.jpg").write_bytes(b"img2")
+
+        bundle_path = create_upload_bundle(
+            output_dir=output_dir,
+            csv_path=csv_path,
+            media_dir=media_dir,
+            project_id=14,
+        )
+
+        assert bundle_path.exists()
+        assert bundle_path.parent == output_dir
+        with zipfile.ZipFile(bundle_path) as zf:
+            names = set(zf.namelist())
+            assert "candidate_results.csv" in names
+            assert "media/100_sig.jpg" in names
+            assert "media/100_page.jpg" in names
+            # ZIP_STORED = no compression
+            for info in zf.infolist():
+                assert info.compress_type == zipfile.ZIP_STORED
+
+    def test_bundle_filename_has_project_id_and_timestamp(self, tmp_path):
+        output_dir = tmp_path / "output"
+        output_dir.mkdir()
+        csv_path = output_dir / "candidate_results.csv"
+        csv_path.write_text("a,b\n1,2\n")
+
+        bundle_path = create_upload_bundle(
+            output_dir=output_dir,
+            csv_path=csv_path,
+            media_dir=output_dir / "media",
+            project_id=42,
+        )
+
+        assert re.fullmatch(
+            r"results_export_p42_\d{8}_\d{6}\.zip", bundle_path.name
+        )
+
+    def test_bundle_handles_missing_media_dir(self, tmp_path):
+        output_dir = tmp_path / "output"
+        output_dir.mkdir()
+        csv_path = output_dir / "candidate_results.csv"
+        csv_path.write_text("a,b\n1,2\n")
+
+        bundle_path = create_upload_bundle(
+            output_dir=output_dir,
+            csv_path=csv_path,
+            media_dir=output_dir / "media",
+            project_id=1,
+        )
+
+        assert bundle_path.exists()
+        with zipfile.ZipFile(bundle_path) as zf:
+            names = set(zf.namelist())
+            assert names == {"candidate_results.csv"}
+
+    def test_bundle_handles_empty_media_dir(self, tmp_path):
+        output_dir = tmp_path / "output"
+        output_dir.mkdir()
+        csv_path = output_dir / "candidate_results.csv"
+        csv_path.write_text("a,b\n1,2\n")
+        media_dir = output_dir / "media"
+        media_dir.mkdir()
+
+        bundle_path = create_upload_bundle(
+            output_dir=output_dir,
+            csv_path=csv_path,
+            media_dir=media_dir,
+            project_id=1,
+        )
+
+        with zipfile.ZipFile(bundle_path) as zf:
+            assert set(zf.namelist()) == {"candidate_results.csv"}
+
+
 class TestCLI:
     def test_with_center_ids(self, output_dir):
         zip_bytes = make_test_zip(100)
@@ -272,6 +355,59 @@ def test_auto_discovers_centers(self, output_dir):
         mock_client.forms.list.assert_called_once_with(project_id=1)
         assert (output_dir / "candidate_results.csv").exists()
 
+    def test_creates_bundle_by_default(self, output_dir):
+        zip_bytes = make_test_zip(100)
+
+        mock_response = MagicMock()
+        mock_response.content = zip_bytes
+        mock_response.raise_for_status = MagicMock()
+
+        mock_client = MagicMock()
+        mock_client.get.return_value = mock_response
+        mock_client.__enter__ = MagicMock(return_value=mock_client)
+        mock_client.__exit__ = MagicMock(return_value=False)
+
+        runner = CliRunner()
+        with patch("src.download_results_forms.Client", return_value=mock_client):
+            result = runner.invoke(
+                main,
+                ["--project-id=14", "--output-dir", str(output_dir), "100"],
+            )
+
+        assert result.exit_code == 0
+        bundles = list(output_dir.glob("results_export_p14_*.zip"))
+        assert len(bundles) == 1
+
+    def test_bundle_false_skips_bundle(self, output_dir):
+        zip_bytes = make_test_zip(100)
+
+        mock_response = MagicMock()
+        mock_response.content = zip_bytes
+        mock_response.raise_for_status = MagicMock()
+
+        mock_client = MagicMock()
+        mock_client.get.return_value = mock_response
+        mock_client.__enter__ = MagicMock(return_value=mock_client)
+        mock_client.__exit__ = MagicMock(return_value=False)
+
+        runner = CliRunner()
+        with patch("src.download_results_forms.Client", return_value=mock_client):
+            result = runner.invoke(
+                main,
+                [
+                    "--project-id=14",
+                    "--output-dir",
+                    str(output_dir),
+                    "--bundle=false",
+                    "100",
+                ],
+            )
+
+        assert result.exit_code == 0
+        assert (output_dir / "candidate_results.csv").exists()
+        bundles = list(output_dir.glob("results_export_p*.zip"))
+        assert len(bundles) == 0
+
     def test_pyodk_error_exits_gracefully(self, output_dir):
         mock_client = MagicMock()
         mock_client.__enter__ = MagicMock(return_value=mock_client)