Merge pull request #7 from ihmeuw/feature/rmbarber/v7-supplemental

use v7.1 and experiment 101; for gbd_2023 raking, set 2024 to end year

Author: rmbarber

src/rra_population_model/constants.py

@@ -27,7 +27,7 @@ def to_list(cls) -> list[str]:
 
 class BuiltVersion(BaseModel):
     provider: Literal["ghsl", "microsoft"]
-    version: Literal["v6", "v7", "r2023a"]
+    version: Literal["v6", "v7", "v7_1", "v7_e101", "r2023a"]
     time_points: list[str]
     measures: list[str]
 
@@ -72,9 +72,6 @@ def time_points_float(self) -> list[float]:
     "microsoft_v7": BuiltVersion(
         provider="microsoft",
         version="v7",
-        # time_points=[
-        #     f"{y}q{q}" for y, q in itertools.product(range(2020, 2025), range(1, 5))
-        # ][1:-2],
         time_points=[
             f"{y}q{q}" for y, q in itertools.product(range(2020, 2024), range(1, 5))
         ][1:],
@@ -83,6 +80,24 @@ def time_points_float(self) -> list[float]:
             "height",
         ],
     ),
+    "microsoft_v7_1": BuiltVersion(
+        provider="microsoft",
+        version="v7_1",
+        time_points=['2020q2', '2024q2'],
+        measures=[
+            "density",
+            "height",
+        ],
+    ),
+    "microsoft_v7_e101": BuiltVersion(
+        provider="microsoft",
+        version="v7_e101",
+        time_points=['2023q4'],
+        measures=[
+            "density",
+            "height",
+        ],
+    ),
 }
 
 DENOMINATORS = []

src/rra_population_model/model/train/runner.py

@@ -110,7 +110,7 @@ def train(
     queue: str,
 ) -> None:
     pm_data = PopulationModelData(output_dir)
-    today, last_version = utils.get_last_run_version(output_dir)
+    today, last_version = utils.get_last_run_version(pm_data.model_root(resolution))
     node_args = []
     for i, (denom, ntl) in enumerate(itertools.product(denominator, ntl_option)):
         version = f"{today}.{last_version + i + 1:03d}"
@@ -132,7 +132,7 @@ def train(
             "queue": queue,
             "cores": 1,
             "memory": "240G",
-            "runtime": "120m",
+            "runtime": "150m",
             "project": "proj_rapidresponse",
         },
         log_root=pm_data.log_dir("model_train"),

src/rra_population_model/model_prep/features/built.py

@@ -132,7 +132,7 @@ def generate_derived_measures(
         if self.built_version.name == "ghsl_r2023a":
             # No derived measures for GHSL
             return {}
-        elif self.built_version.name in ["microsoft_v6", "microsoft_v7"]:
+        elif self.built_version.name.startswith("microsoft"):
             return _generate_microsoft_derived_measures(pm_data, self.feature_metadata, self.built_version.name)
         else:
             msg = f"Unknown built version: {self.built_version.name}"
@@ -269,6 +269,16 @@ def _generate_microsoft_derived_measures(
             "height": "microsoft_v7_height",
             "p_residential": "ghsl_r2023a_proportion_residential",
         },
+        "microsoft_v7_1": {
+            "density": "microsoft_v7_1_density",
+            "height": "microsoft_v7_1_height",
+            "p_residential": "ghsl_r2023a_proportion_residential",
+        },
+        "microsoft_v7_e101": {
+            "density": "microsoft_v7_e101_density",
+            "height": "microsoft_v7_e101_height",
+            "p_residential": "ghsl_r2023a_proportion_residential",
+        },
     }[built_version_name]
     density = pm_data.load_feature(
         feature_name=feature_dict["density"],

src/rra_population_model/model_prep/features/runner.py

@@ -15,10 +15,13 @@
 from rra_population_model.model_prep.features.metadata import get_feature_metadata
 from rra_population_model.model_prep.features.ntl import process_ntl
 
-# GHSL first, as we need the height and residential mask for msft
+# GHSL first, as we need the residential mask for msft
 BUILT_VERSIONS = [
     # pmc.BUILT_VERSIONS["ghsl_r2023a"],
-    pmc.BUILT_VERSIONS["microsoft_v7"],
+    # pmc.BUILT_VERSIONS["microsoft_v6"],
+    # pmc.BUILT_VERSIONS["microsoft_v7"],
+    pmc.BUILT_VERSIONS["microsoft_v7_1"],
+    pmc.BUILT_VERSIONS["microsoft_v7_e101"],
 ]
 
 
@@ -172,8 +175,8 @@ def features(
         task_resources={
             "queue": queue,
             "cores": 1,
-            "memory": "15G",
-            "runtime": "30m",
+            "memory": "5G",
+            "runtime": "10m",
             "project": "proj_rapidresponse",
             "constraints": "archive",
         },

src/rra_population_model/model_prep/training_data/runner.py

@@ -154,7 +154,7 @@ def training_data(
             "queue": queue,
             "cores": 1,
             "memory": "30G",
-            "runtime": "60m",
+            "runtime": "30m",
             "project": "proj_rapidresponse",
         },
         max_attempts=5,

src/rra_population_model/postprocess/rake_itu.py

@@ -28,7 +28,10 @@ def load_admin_populations(
     # Interpolate the time point population
     if "q" in time_point:
         year, quarter = (int(s) for s in time_point.split("q"))
-        next_year = min(year + 1, 2100)
+        if RAKING_VERSION == "gbd_2023":
+            next_year = min(year + 1, 2024)
+        else:
+            next_year = min(year + 1, 2100)
         weight = (int(quarter) - 1) / 4
 
         prior_year_pop = all_pop.loc[year]

src/rra_population_model/postprocess/raking_factors/runner.py

@@ -29,7 +29,10 @@ def load_admin_populations(
     # Interpolate the time point population
     if "q" in time_point:
         year, quarter = (int(s) for s in time_point.split("q"))
-        next_year = min(year + 1, 2100)
+        if RAKING_VERSION == "gbd_2023":
+            next_year = min(year + 1, 2024)
+        else:
+            next_year = min(year + 1, 2100)
         weight = (int(quarter) - 1) / 4
 
         prior_year_pop = all_pop.loc[year]