Add --clean-frs-base for per-year clean FRS, update Modal deployment

- Add --clean-frs-base flag: loads clean CSVs from base/YYYY/ dirs,
  falls back to latest available year + uprating for projected years
- Update Modal app to use single volume with per-year clean FRS
- Simplify upload script for clean FRS base directory structure
- Extract all 30 years of clean CSVs into data/frs_clean_all/

Author: nikhilwoodruff

api/modal_app.py

@@ -3,27 +3,24 @@
 
 Architecture:
   - Rust binary is compiled at image build time (baked into the image layer).
-  - A Modal Volume (`policyengine-uk-frs`) holds the clean FRS microdata CSVs.
-    Upload once with: python api/upload_frs.py <path-to-frs-clean-dir>
+  - Modal Volume (`policyengine-uk-frs`) holds per-year clean FRS CSVs (1994/-2023/).
+    Upload with: python api/upload_frs.py data/frs_clean_all
   - FastAPI app is served via modal.asgi_app().
 
 Deploy:
     modal deploy api/modal_app.py
 
 Serve locally (with hot-reload):
     modal serve api/modal_app.py
-
-Upload FRS data to the volume:
-    python api/upload_frs.py data/frs_clean
 """
 
 import modal
 
 # ---------------------------------------------------------------------------
-# Volume for FRS clean microdata (persons.csv / benunits.csv / households.csv)
+# Volumes
 # ---------------------------------------------------------------------------
 frs_volume = modal.Volume.from_name("policyengine-uk-frs", create_if_missing=True)
-FRS_MOUNT_PATH = "/data/frs_clean"
+FRS_MOUNT = "/data/frs_clean"
 
 # ---------------------------------------------------------------------------
 # Image — Debian base, install Rust toolchain, clone repo, compile binary
@@ -38,7 +35,6 @@
     )
     .pip_install("fastapi>=0.115", "uvicorn[standard]>=0.30", "pydantic>=2.0")
     # Copy the repo source into the image (exclude FRS data — it stays on the Volume)
-    # The .dockerignore / ignore list ensures data/frs_clean never enters the image.
     .add_local_dir(".", remote_path="/app", copy=True,
                    ignore=["data/", "target/", ".git/", "app/node_modules/", "app/.next/"])
     .run_commands(
@@ -57,7 +53,7 @@
 
 
 # ---------------------------------------------------------------------------
-# FastAPI application (identical logic to api/main.py but paths adjusted)
+# FastAPI application
 # ---------------------------------------------------------------------------
 def _make_fastapi_app():
     import json
@@ -70,9 +66,8 @@ def _make_fastapi_app():
     from pydantic import BaseModel
 
     RUST_BINARY = "policyengine-uk"
-    CLEAN_FRS_DIR = FRS_MOUNT_PATH
-    PARAMETERS_DIR = "/app/parameters"
-    AVAILABLE_YEARS = [2023, 2024, 2025, 2026, 2027, 2028, 2029]
+    FRS_BASE_DIR = FRS_MOUNT
+    AVAILABLE_YEARS = list(range(1994, 2030))
 
     fastapi_app = FastAPI(title="PolicyEngine UK API")
 
@@ -87,8 +82,8 @@ def _make_fastapi_app():
     params_cache: dict[int, dict] = {}
 
     def _data_args() -> list[str]:
-        if os.path.isdir(CLEAN_FRS_DIR) and os.listdir(CLEAN_FRS_DIR):
-            return ["--clean-frs", CLEAN_FRS_DIR]
+        if os.path.isdir(FRS_BASE_DIR) and os.listdir(FRS_BASE_DIR):
+            return ["--clean-frs-base", FRS_BASE_DIR]
         return []
 
     def run_simulation(year: int, reform_json: Optional[str] = None) -> dict:
@@ -133,7 +128,7 @@ async def cache_baselines():
     REFORM_SECTIONS = [
         "income_tax", "national_insurance", "universal_credit",
         "child_benefit", "benefit_cap", "housing_benefit",
-        "tax_credits", "scottish_child_payment",
+        "tax_credits", "council_tax_reduction", "scottish_child_payment",
         "pension_credit", "state_pension",
     ]
 
@@ -146,6 +141,7 @@ class SimulateRequest(BaseModel):
         benefit_cap: Optional[dict[str, Any]] = None
         housing_benefit: Optional[dict[str, Any]] = None
         tax_credits: Optional[dict[str, Any]] = None
+        council_tax_reduction: Optional[dict[str, Any]] = None
         scottish_child_payment: Optional[dict[str, Any]] = None
         pension_credit: Optional[dict[str, Any]] = None
         state_pension: Optional[dict[str, Any]] = None
@@ -159,6 +155,7 @@ class SimulateMultiYearRequest(BaseModel):
         benefit_cap: Optional[dict[str, Any]] = None
         housing_benefit: Optional[dict[str, Any]] = None
         tax_credits: Optional[dict[str, Any]] = None
+        council_tax_reduction: Optional[dict[str, Any]] = None
         scottish_child_payment: Optional[dict[str, Any]] = None
         pension_credit: Optional[dict[str, Any]] = None
         state_pension: Optional[dict[str, Any]] = None
@@ -219,19 +216,19 @@ async def health():
         return {
             "status": "ok",
             "binary": bool(shutil.which("policyengine-uk")),
-            "frs_data": os.path.isdir(CLEAN_FRS_DIR) and bool(os.listdir(CLEAN_FRS_DIR)),
-            "frs_path": CLEAN_FRS_DIR,
+            "frs_data": os.path.isdir(FRS_BASE_DIR) and bool(os.listdir(FRS_BASE_DIR)),
+            "cached_years": sorted(baseline_cache.keys()),
         }
 
     return fastapi_app
 
 
 @app.function(
-    volumes={FRS_MOUNT_PATH: frs_volume},
-    # Startup caches all 7 year baselines — give it enough RAM
-    memory=4096,
+    volumes={FRS_MOUNT: frs_volume},
+    # Startup caches 36 year baselines — needs time and memory
+    memory=8192,
     cpu=4,
-    timeout=300,
+    timeout=600,
     # EU West (Ireland) for lower latency from UK callers
     region="eu-west-1",
 )

api/upload_frs.py

@@ -1,11 +1,16 @@
 """
-Upload clean FRS CSV files to the Modal Volume.
+Upload clean FRS data to the Modal Volume.
 
 Usage:
-    python api/upload_frs.py <path-to-frs-clean-dir>
+    python api/upload_frs.py data/frs_clean_all
 
-The directory should contain: persons.csv, benunits.csv, households.csv
-(generated by: policyengine-uk --extract-frs <output-dir> --frs <raw-frs-dir>)
+The directory should contain per-year subdirectories (1994/, 1995/, ..., 2023/)
+each with persons.csv, benunits.csv, households.csv.
+
+Generate with:
+    for year in $(seq 1994 2023); do
+        ./target/release/policyengine-uk-rust --year $year --frs <tab-dir> --extract-frs data/frs_clean_all/$year
+    done
 
 This only needs to be run once (or when the FRS data changes).
 """
@@ -14,36 +19,34 @@
 from pathlib import Path
 import modal
 
-FRS_VOLUME_NAME = "policyengine-uk-frs"
-REMOTE_PATH = "/data/frs_clean"
-EXPECTED_FILES = {"persons.csv", "benunits.csv", "households.csv"}
+VOLUME_NAME = "policyengine-uk-frs"
 
 
 def upload(local_dir: Path) -> None:
-    missing = EXPECTED_FILES - {f.name for f in local_dir.iterdir() if f.is_file()}
-    if missing:
-        print(f"Error: missing files in {local_dir}: {missing}", file=sys.stderr)
+    year_dirs = sorted(d for d in local_dir.iterdir() if d.is_dir() and d.name.isdigit())
+    if not year_dirs:
+        print(f"Error: no year directories found in {local_dir}", file=sys.stderr)
         sys.exit(1)
 
-    volume = modal.Volume.from_name(FRS_VOLUME_NAME, create_if_missing=True)
+    volume = modal.Volume.from_name(VOLUME_NAME, create_if_missing=True)
+    print(f"Uploading {len(year_dirs)} years from {local_dir} → Modal Volume '{VOLUME_NAME}'")
 
-    # Files are stored at the volume root (e.g. "persons.csv").
-    # The volume is mounted at FRS_MOUNT_PATH in the container, so
-    # container path /data/frs_clean/persons.csv == volume root persons.csv.
-    print(f"Uploading FRS clean CSVs from {local_dir} → Modal Volume '{FRS_VOLUME_NAME}' (root)")
     with volume.batch_upload(force=True) as batch:
-        for csv_file in local_dir.glob("*.csv"):
-            batch.put_file(str(csv_file), csv_file.name)
-            print(f"  {csv_file.name} → /{csv_file.name}")
+        for year_dir in year_dirs:
+            csvs = list(year_dir.glob("*.csv"))
+            if not csvs:
+                print(f"  WARNING: no CSVs in {year_dir.name}, skipping")
+                continue
+            print(f"  {year_dir.name}: {len(csvs)} files")
+            for f in csvs:
+                batch.put_file(str(f), f"{year_dir.name}/{f.name}")
 
-    print("Done. Volume contents:")
-    for entry in volume.listdir("/"):
-        print(f"  {entry.path}")
+    print("Done.")
 
 
 if __name__ == "__main__":
     if len(sys.argv) != 2:
-        print(f"Usage: python {sys.argv[0]} <frs-clean-dir>", file=sys.stderr)
+        print(f"Usage: python {sys.argv[0]} <frs-clean-base-dir>", file=sys.stderr)
         sys.exit(1)
 
     local_dir = Path(sys.argv[1])

src/main.rs

@@ -62,6 +62,12 @@ struct Cli {
     #[arg(long)]
     clean_frs: Option<PathBuf>,
 
+    /// Base directory containing per-year clean FRS subdirectories (YYYY/).
+    /// Each subdirectory has persons.csv, benunits.csv, households.csv.
+    /// Falls back to latest available year and uprates if needed.
+    #[arg(long)]
+    clean_frs_base: Option<PathBuf>,
+
     /// Output format: "pretty" (default) or "json" for machine-readable output
     #[arg(long, default_value = "pretty")]
     output: String,
@@ -256,7 +262,27 @@ fn main() -> anyhow::Result<()> {
     }
 
     // Load dataset
-    let dataset = if let Some(clean_path) = &cli.clean_frs {
+    let dataset = if let Some(base) = &cli.clean_frs_base {
+        // Per-year clean FRS directories: base/YYYY/
+        let year_dir = base.join(cli.year.to_string());
+        if year_dir.is_dir() {
+            if !json_mode { println!("  {} Loading clean FRS {}/{}...", "▸".bright_cyan(), cli.year, (cli.year + 1) % 100); }
+            load_clean_frs(&year_dir)?
+        } else {
+            // Find latest available year and uprate
+            let latest = (1994..=cli.year).rev()
+                .find(|y| base.join(y.to_string()).is_dir())
+                .ok_or_else(|| anyhow::anyhow!("No clean FRS data found in {}", base.display()))?;
+            if !json_mode {
+                println!("  {} Loading clean FRS {}/{} and uprating to {}/{}...",
+                    "▸".bright_cyan(), latest, (latest + 1) % 100,
+                    cli.year, (cli.year + 1) % 100);
+            }
+            let mut ds = load_clean_frs(&base.join(latest.to_string()))?;
+            ds.uprate_to(cli.year);
+            ds
+        }
+    } else if let Some(clean_path) = &cli.clean_frs {
         if !json_mode { println!("  {} Loading clean FRS from {}...", "▸".bright_cyan(), clean_path.display()); }
         load_clean_frs(clean_path)?
     } else if let Some(frs_path) = &cli.frs {