Refactor to use new simplified pinval.vw_assessment_card data model

hugo/layouts/ineligible.html

@@ -25,10 +25,10 @@ <h3>Report not available</h3>
         </p>
         {{ else if eq .Params.reason_report_ineligible "non_regression_class" }}
         <p>
-          This property has class code <strong>{{ .Params.parcel_class }}</strong>.
-          {{ if .Params.parcel_class_description }}
+          This property has class code <strong>{{ .Params.char_class }}</strong>.
+          {{ if .Params.char_class_desc }}
             Properties with this code should fit the description
-            <strong>"{{ .Params.parcel_class_description }}"</strong>.
+            <strong>"{{ .Params.char_class_desc }}"</strong>.
           {{ end }}
         </p>
         <p>
@@ -41,7 +41,7 @@ <h3>Report not available</h3>
           The Assessor did not reassess this home in
           {{ .Params.assessment_year }}. In {{ .Params.assessment_year }},
           the Assessor reassessed the <strong>{{ title .Params.assessment_triad_name }}</strong>
-          triad, but this home is in the <strong>{{ title .Params.parcel_triad_name }}</strong>
+          triad, but this home is in the <strong>{{ title .Params.meta_triad_name }}</strong>
           triad.
         </p>
         {{ else if eq .Params.reason_report_ineligible "missing_card" }}

hugo/layouts/report.html

@@ -181,7 +181,7 @@ <h5>What information does this report contain?</h5>
           >
             {{ template "card-content"
             (dict "card" $card
-                  "var_labels" $.Params.var_labels
+                  "Params" $.Params
                   "is_multicard" true)}}
           </div>
           {{ end }}
@@ -192,7 +192,7 @@ <h5>What information does this report contain?</h5>
         {{ $card := index .Params.cards 0 }}
         {{ template "card-content"
           (dict "card" $card
-          "var_labels" .Params.var_labels
+          "Params" .Params
           "is_multicard" false) }}
       {{ end }}
 
@@ -356,7 +356,7 @@ <h2 class="mb-3">Top 5 Most Important Sales</h2>
         <!-- Regular rows -->
         {{ range $char := .card.predictors }}
         <tr>
-          <td class="char-name">{{ index $.var_labels $char }}</td>
+          <td class="char-name">{{ index $.Params.var_labels $char }}</td>
           <td>{{ index $.card.chars $char }}</td>
 
           {{ range $comp := $.card.comps }}

scripts/generate_pinval/constants.py

@@ -1,5 +1,13 @@
 """Constants that are shared between scripts in this module"""
 
 # Temporary solution for run_id mapping, a problem that occurs when the model run_id
-# differs between the model values and the comps
-RUN_ID_MAP = {"2025-02-11-charming-eric": "2025-04-25-fancy-free-billy"}
+# differs between the model values and the comps.
+# Currently we don't need more than one run because comps and cards are using the same
+# final run ID, but we're keeping this map around with the expectation that we may
+# need it for a more permanent fix
+RUN_ID_MAP = {"2025-02-11-charming-eric": "2025-02-11-charming-eric"}
+
+# It's helpful to factor these tables out into shared constants because we often
+# need to switch to dev tables for testing
+PINVAL_ASSESSMENT_CARD_TABLE = "pinval.vw_assessment_card"
+PINVAL_COMP_TABLE = "pinval.vw_comp"

scripts/generate_pinval/generate_pinval.py

@@ -36,7 +36,7 @@
 from pyathena import connect
 from pyathena.pandas.cursor import PandasCursor
 
-from constants import RUN_ID_MAP
+import constants
 
 
 # Argparse interface
@@ -54,7 +54,7 @@ def parse_args() -> argparse.Namespace:
         "--run-id",
         required=True,
         choices=list(
-            RUN_ID_MAP.keys()
+            constants.RUN_ID_MAP.keys()
         ),  # Temporarily limits run_ids to those in the map
         help="Model run‑ID used by the Athena PINVAL tables (e.g. 2025-02-11-charming-eric)",
     )
@@ -172,8 +172,8 @@ def build_front_matter(
         "final_model_run_date": pd.to_datetime(tp["final_model_run_date"]).strftime(
             "%B %d, %Y"
         ),
-        "pin": tp["pin"],
-        "pin_pretty": pin_pretty(tp["pin"]),
+        "pin": tp["meta_pin"],
+        "pin_pretty": pin_pretty(tp["meta_pin"]),
         "pred_pin_final_fmv_round": tp["pred_pin_final_fmv_round"],
         "cards": [],
         "var_labels": {k: pretty_fn(k) for k in preds_cleaned},
@@ -183,11 +183,14 @@ def build_front_matter(
     # will pass the doc some info to help explain why the parcel is ineligible
     if not tp["is_report_eligible"]:
         front["layout"] = "ineligible"
-        front["reason_report_ineligible"] = tp["reason_report_ineligible"]
-        front["assessment_triad_name"] = tp["assessment_triad_name"]
-        front["parcel_class"] = tp["parcel_class"]
-        front["parcel_class_description"] = tp["parcel_class_description"]
-        front["parcel_triad_name"] = tp["parcel_triad_name"]
+        for attr in [
+            "reason_report_ineligible",
+            "assessment_triad_name",
+            "char_class",
+            "char_class_desc",
+            "meta_triad_name",
+        ]:
+            front[attr] = tp[attr]
         return front
 
     # Per card
@@ -252,7 +255,7 @@ def build_front_matter(
                     for k, v in {
                         "property_address": card_df["property_address"],
                         "municipality": card_df.get("loc_tax_municipality_name"),
-                        "township": tp["parcel_township_name"],
+                        "township": tp["meta_township_name"],
                         "meta_nbhd_code": card_df["meta_nbhd_code"],
                         "loc_school_elementary_district_name": card_df.get(
                             "school_elementary_district_name"
@@ -505,33 +508,27 @@ def main() -> None:
 
     assessment_year = assessment_year_df.iloc[0]["assessment_year"]
 
-    # Use `model_run_id` instead of `run_id` because `run_id` comes from
-    # `model.assessment_card` and so is not present for ineligible PINs,
-    # but we want to query those PINs so we can generate error pages for them
-    assessment_clauses = ["model_run_id = %(run_id)s"]
+    assessment_clauses = ["run_id = %(run_id)s"]
     params_assessment = {"run_id": args.run_id}
 
     # Shard by township **only** in the assessment query
     if args.township:
-        # Similar to `model_run_id` above, use `parcel_township_code` here
-        # instead of `meta_township_code` because the latter will be null
-        # for ineligible PINs
-        assessment_clauses.append("parcel_township_code = %(township)s")
+        assessment_clauses.append("meta_township_code = %(township)s")
         params_assessment["township"] = args.township
 
     if args.pin:
         pins: list[str] = list(set(args.pin))  # de-dupe
         pin_params = {f"pin{i}": p for i, p in enumerate(pins)}
         placeholders = ",".join(f"%({k})s" for k in pin_params)
 
-        assessment_clauses.append(f"pin IN ({placeholders})")
+        assessment_clauses.append(f"meta_pin IN ({placeholders})")
         params_assessment = {**params_assessment, **pin_params}
 
     where_assessment = " AND ".join(assessment_clauses)
 
     assessment_sql = f"""
         SELECT *
-        FROM pinval.vw_assessment_card
+        FROM {constants.PINVAL_ASSESSMENT_CARD_TABLE}
         WHERE {where_assessment}
     """
 
@@ -548,18 +545,18 @@ def main() -> None:
         )
 
     # Get the comps
-    if (comps_run_id := RUN_ID_MAP.get(args.run_id)) is None:
+    if (comps_run_id := constants.RUN_ID_MAP.get(args.run_id)) is None:
         raise ValueError(f"No comps run ID found for assessment run ID {args.run_id}")
 
     comps_sql = f"""
         SELECT comp.*
-        FROM pinval.vw_comp AS comp
+        FROM {constants.PINVAL_COMP_TABLE} AS comp
         INNER JOIN (
-            SELECT DISTINCT pin
-            FROM pinval.vw_assessment_card
+            SELECT DISTINCT meta_pin
+            FROM {constants.PINVAL_ASSESSMENT_CARD_TABLE}
             WHERE {where_assessment}
         ) AS card
-          ON comp.pin = card.pin
+          ON comp.pin = card.meta_pin
         WHERE comp.run_id = %(comps_run_id)s
     """
 
@@ -606,7 +603,7 @@ def pretty(k: str) -> str:
     start_time_dict_groupby = time.time()
 
     # Group dfs by PIN in dict for theoretically faster access
-    df_assessments_by_pin = df_assessment_all.groupby("pin")
+    df_assessments_by_pin = df_assessment_all.groupby("meta_pin")
     df_comps_by_pin = (
         {} if df_comps_all.empty else dict(tuple(df_comps_all.groupby("pin")))
     )

scripts/generate_pinval/resolve_model_metadata.py

@@ -15,6 +15,8 @@
 from pyathena import connect
 from pyathena.cursor import DictCursor
 
+import constants
+
 
 def parse_args() -> argparse.Namespace:
     """Parse command‑line arguments and perform basic validation"""
@@ -104,11 +106,11 @@ def get_township_codes(
             for row in connect(region_name="us-east-1")
             .cursor()
             .execute(
-                """
-                    SELECT DISTINCT parcel_township_code
-                    FROM pinval.vw_assessment_card
-                    WHERE model_run_id = %(run_id)s
-                    ORDER BY parcel_township_code
+                f"""
+                    SELECT DISTINCT meta_township_code
+                    FROM {constants.PINVAL_ASSESSMENT_CARD_TABLE}
+                    WHERE run_id = %(run_id)s
+                    ORDER BY meta_township_code
                 """,
                 {"run_id": run_id},
             )