Deprecate SIMULATE_CHOOSER_COLUMNS and LOGSUM_CHOOSER_COLUMNS settings

Author: Copilot

activitysim/abm/models/location_choice.py

@@ -242,14 +242,7 @@ def location_sample(
     chunk_tag,
     trace_label,
 ):
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-    # Drop this when PR #1017 is merged
-    if ("household_id" not in chooser_columns) and (
-        "household_id" in persons_merged.columns
-    ):
-        chooser_columns = chooser_columns + ["household_id"]
-    choosers = persons_merged[chooser_columns]
+    choosers = persons_merged
 
     # create wrapper with keys for this lookup - in this case there is a home_zone_id in the choosers
     # and a zone_id in the alternatives which get merged during interaction
@@ -441,17 +434,7 @@ def location_presample(
         HOME_TAZ in persons_merged
     )  # 'TAZ' should already be in persons_merged from land_use
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    # FIXME we don't actually require that land_use provide a TAZ crosswalk
-    # FIXME maybe we should add it for multi-zone (from maz_taz) if missing?
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-    chooser_columns = [HOME_TAZ if c == HOME_MAZ else c for c in chooser_columns]
-    # Drop this when PR #1017 is merged
-    if ("household_id" not in chooser_columns) and (
-        "household_id" in persons_merged.columns
-    ):
-        chooser_columns = chooser_columns + ["household_id"]
-    choosers = persons_merged[chooser_columns]
+    choosers = persons_merged
 
     # create wrapper with keys for this lookup - in this case there is a HOME_TAZ in the choosers
     # and a DEST_TAZ in the alternatives which get merged during interaction
@@ -627,11 +610,6 @@ def run_location_logsums(
         mandatory=False,
     )
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    persons_merged_df = logsum.filter_chooser_columns(
-        persons_merged_df, logsum_settings, model_settings
-    )
-
     logger.info(f"Running {trace_label} with {len(location_sample_df.index)} rows")
 
     choosers = location_sample_df.join(persons_merged_df, how="left")
@@ -691,14 +669,7 @@ def run_location_simulate(
     """
     assert not persons_merged.empty
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-    # Drop this when PR #1017 is merged
-    if ("household_id" not in chooser_columns) and (
-        "household_id" in persons_merged.columns
-    ):
-        chooser_columns = chooser_columns + ["household_id"]
-    choosers = persons_merged[chooser_columns]
+    choosers = persons_merged
 
     alt_dest_col_name = model_settings.ALT_DEST_COL_NAME
 

activitysim/abm/models/school_escorting.py

@@ -3,10 +3,12 @@
 from __future__ import annotations
 
 import logging
+import warnings
 from typing import Any, Literal
 
 import numpy as np
 import pandas as pd
+from pydantic import field_validator
 
 from activitysim.abm.models.util import school_escort_tours_trips
 from activitysim.core import (
@@ -335,7 +337,26 @@ class SchoolEscortSettings(BaseLogitComponentSettings, extra="forbid"):
     GENDER_WEIGHT: float = 10.0
     AGE_WEIGHT: float = 1.0
 
-    SIMULATE_CHOOSER_COLUMNS: list[str] | None = None
+    SIMULATE_CHOOSER_COLUMNS: Any | None = None
+    """Was used to help reduce the memory needed for the model.
+
+    This setting is now obsolete and does nothing. Its functionality has been
+    replaced by :func:`activitysim.core.util.drop_unused_columns`.
+
+    .. deprecated:: 1.4
+    """
+
+    @field_validator("SIMULATE_CHOOSER_COLUMNS", mode="before")
+    @classmethod
+    def _deprecate_simulate_chooser_columns(cls, value):
+        if value is not None:
+            warnings.warn(
+                "SIMULATE_CHOOSER_COLUMNS is deprecated and no longer used, "
+                "unused columns are now dropped automatically",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+        return None
 
     SPEC: None = None
     """The school escort model does not use this setting."""
@@ -465,17 +486,6 @@ def school_escorting(
         # else:
         #     locals_dict.pop("_sharrow_skip", None)
 
-        # reduce memory by limiting columns if selected columns are supplied
-        chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-        if chooser_columns is not None:
-            # Drop this when PR #1017 is merged
-            if ("household_id" not in chooser_columns) and (
-                "household_id" in choosers.columns
-            ):
-                chooser_columns = chooser_columns + ["household_id"]
-            chooser_columns = chooser_columns + participant_columns
-            choosers = choosers[chooser_columns]
-
         # add previous data to stage
         if stage_num >= 1:
             choosers = add_prev_choices_to_choosers(

activitysim/abm/models/util/logsums.py

@@ -7,7 +7,6 @@
 import pandas as pd
 
 from activitysim.core import config, expressions, los, simulate, tracing, workflow
-from activitysim.core.configuration import PydanticBase
 from activitysim.core.configuration.logit import (
     TourLocationComponentSettings,
     TourModeComponentSettings,
@@ -16,41 +15,6 @@
 logger = logging.getLogger(__name__)
 
 
-def filter_chooser_columns(
-    choosers, logsum_settings: dict | PydanticBase, model_settings: dict | PydanticBase
-):
-    try:
-        chooser_columns = logsum_settings.LOGSUM_CHOOSER_COLUMNS
-    except AttributeError:
-        chooser_columns = logsum_settings.get("LOGSUM_CHOOSER_COLUMNS", [])
-
-    if (
-        isinstance(model_settings, dict)
-        and "CHOOSER_ORIG_COL_NAME" in model_settings
-        and model_settings["CHOOSER_ORIG_COL_NAME"] not in chooser_columns
-    ):
-        chooser_columns.append(model_settings["CHOOSER_ORIG_COL_NAME"])
-    if (
-        isinstance(model_settings, PydanticBase)
-        and hasattr(model_settings, "CHOOSER_ORIG_COL_NAME")
-        and model_settings.CHOOSER_ORIG_COL_NAME
-        and model_settings.CHOOSER_ORIG_COL_NAME not in chooser_columns
-    ):
-        chooser_columns.append(model_settings.CHOOSER_ORIG_COL_NAME)
-
-    missing_columns = [c for c in chooser_columns if c not in choosers]
-    if missing_columns:
-        logger.debug(
-            "logsum.filter_chooser_columns missing_columns %s" % missing_columns
-        )
-
-    # ignore any columns not appearing in choosers df
-    chooser_columns = [c for c in chooser_columns if c in choosers]
-
-    choosers = choosers[chooser_columns]
-    return choosers
-
-
 def compute_location_choice_logsums(
     state: workflow.State,
     choosers: pd.DataFrame,

activitysim/abm/models/util/tour_destination.py

@@ -691,23 +691,9 @@ def run_destination_sample(
     chunk_size,
     trace_label,
 ):
-    # FIXME - MEMORY HACK - only include columns actually used in spec (omit them pre-merge)
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-
     # if special person id is passed
     chooser_id_column = model_settings.CHOOSER_ID_COLUMN
 
-    # Drop this when PR #1017 is merged
-    if ("household_id" not in chooser_columns) and (
-        "household_id" in persons_merged.columns
-    ):
-        chooser_columns = chooser_columns + ["household_id"]
-    persons_merged = persons_merged[
-        [c for c in persons_merged.columns if c in chooser_columns]
-    ]
-    tours = tours[
-        [c for c in tours.columns if c in chooser_columns or c == chooser_id_column]
-    ]
     choosers = pd.merge(
         tours, persons_merged, left_on=chooser_id_column, right_index=True, how="left"
     )
@@ -805,11 +791,6 @@ def run_destination_logsums(
 
     chunk_tag = "tour_destination.logsums"
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    persons_merged = logsum.filter_chooser_columns(
-        persons_merged, logsum_settings, model_settings
-    )
-
     # merge persons into tours
     choosers = pd.merge(
         destination_sample,
@@ -872,23 +853,9 @@ def run_destination_simulate(
         coefficients_file_name=model_settings.COEFFICIENTS,
     )
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec (omit them pre-merge)
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-
     # if special person id is passed
     chooser_id_column = model_settings.CHOOSER_ID_COLUMN
 
-    # Drop this when PR #1017 is merged
-    if ("household_id" not in chooser_columns) and (
-        "household_id" in persons_merged.columns
-    ):
-        chooser_columns = chooser_columns + ["household_id"]
-    persons_merged = persons_merged[
-        [c for c in persons_merged.columns if c in chooser_columns]
-    ]
-    tours = tours[
-        [c for c in tours.columns if c in chooser_columns or c == chooser_id_column]
-    ]
     choosers = pd.merge(
         tours, persons_merged, left_on=chooser_id_column, right_index=True, how="left"
     )

activitysim/abm/models/util/tour_od.py

@@ -737,12 +737,6 @@ def run_od_sample(
     )
 
     choosers = tours
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-    # Drop this when PR #1017 is merged
-    if ("household_id" not in chooser_columns) and ("household_id" in choosers.columns):
-        chooser_columns = chooser_columns + ["household_id"]
-    choosers = choosers[chooser_columns]
 
     # interaction_sample requires that choosers.index.is_monotonic_increasing
     if not choosers.index.is_monotonic_increasing:
@@ -820,11 +814,6 @@ def run_od_logsums(
     dest_id_col = model_settings.DEST_COL_NAME
     tour_od_id_col = get_od_id_col(origin_id_col, dest_id_col)
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    tours_merged_df = logsum.filter_chooser_columns(
-        tours_merged_df, logsum_settings, model_settings
-    )
-
     # merge ods into choosers table
     choosers = od_sample.join(tours_merged_df, how="left")
     choosers[tour_od_id_col] = (
@@ -1000,13 +989,6 @@ def run_od_simulate(
     # merge persons into tours
     choosers = tours
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-    # Drop this when PR #1017 is merged
-    if ("household_id" not in chooser_columns) and ("household_id" in choosers.columns):
-        chooser_columns = chooser_columns + ["household_id"]
-    choosers = choosers[chooser_columns]
-
     # interaction_sample requires that choosers.index.is_monotonic_increasing
     if not choosers.index.is_monotonic_increasing:
         logger.debug(

activitysim/abm/models/util/tour_scheduling.py

@@ -9,7 +9,7 @@
 from activitysim.abm.models.util import vectorize_tour_scheduling as vts
 from activitysim.core import config, estimation, expressions, simulate, workflow
 
-from .vectorize_tour_scheduling import TourModeComponentSettings, TourSchedulingSettings
+from .vectorize_tour_scheduling import TourSchedulingSettings
 
 logger = logging.getLogger(__name__)
 
@@ -24,30 +24,6 @@ def run_tour_scheduling(
     trace_label: str,
 ):
 
-    if model_settings.LOGSUM_SETTINGS:
-        logsum_settings = TourModeComponentSettings.read_settings_file(
-            state.filesystem,
-            str(model_settings.LOGSUM_SETTINGS),
-            mandatory=False,
-        )
-        logsum_columns = logsum_settings.LOGSUM_CHOOSER_COLUMNS
-    else:
-        logsum_columns = []
-
-    # - filter chooser columns for both logsums and simulate
-    model_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-    chooser_columns = logsum_columns + [
-        c for c in model_columns if c not in logsum_columns
-    ]
-
-    # Drop this when PR #1017 is merged
-    if ("household_id" not in chooser_columns) and (
-        "household_id" in persons_merged.columns
-    ):
-        chooser_columns = chooser_columns + ["household_id"]
-
-    persons_merged = expressions.filter_chooser_columns(persons_merged, chooser_columns)
-
     timetable = state.get_injectable("timetable")
 
     # - run preprocessor to annotate choosers

activitysim/abm/models/util/vectorize_tour_scheduling.py

@@ -3,12 +3,14 @@
 from __future__ import annotations
 
 import logging
+import warnings
 from collections import OrderedDict
 from pathlib import Path
 from typing import Any
 
 import numpy as np
 import pandas as pd
+from pydantic import field_validator
 
 from activitysim.abm.models.tour_mode_choice import TourModeComponentSettings
 from activitysim.core import chunk, config, expressions, los, simulate
@@ -43,7 +45,26 @@ class TourSchedulingSettings(LogitComponentSettings, extra="forbid"):
     it is assumed to be an unsegmented preprocessor.  Otherwise, the dict keys
     give the segements.
     """
-    SIMULATE_CHOOSER_COLUMNS: list[str] | None = None
+    SIMULATE_CHOOSER_COLUMNS: Any | None = None
+    """Was used to help reduce the memory needed for the model.
+
+    This setting is now obsolete and does nothing. Its functionality has been
+    replaced by :func:`activitysim.core.util.drop_unused_columns`.
+
+    .. deprecated:: 1.4
+    """
+
+    @field_validator("SIMULATE_CHOOSER_COLUMNS", mode="before")
+    @classmethod
+    def _deprecate_simulate_chooser_columns(cls, value):
+        if value is not None:
+            warnings.warn(
+                "SIMULATE_CHOOSER_COLUMNS is deprecated and no longer used, "
+                "unused columns are now dropped automatically",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+        return None
 
     SPEC_SEGMENTS: dict[str, LogitComponentSettings] = {}
 

activitysim/abm/test/test_misc/test_location_choice_sampling.py

@@ -93,7 +93,6 @@ def fake_choose_maz_for_taz(
         (),
         {
             "ALT_DEST_COL_NAME": "zone_id",
-            "SIMULATE_CHOOSER_COLUMNS": [location_choice.HOME_MAZ],
         },
     )()
     persons_merged = pd.DataFrame(
@@ -222,7 +221,6 @@ def fake_choose_maz_for_taz(
         (),
         {
             "ALT_DEST_COL_NAME": "zone_id",
-            "SIMULATE_CHOOSER_COLUMNS": [location_choice.HOME_MAZ],
         },
     )()
     persons_merged = pd.DataFrame(
@@ -327,7 +325,6 @@ def fake_location_sample(
         (),
         {
             "ALT_DEST_COL_NAME": "zone_id",
-            "SIMULATE_CHOOSER_COLUMNS": [location_choice.HOME_MAZ],
         },
     )()
     persons_merged = pd.DataFrame(

activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py

@@ -122,14 +122,6 @@ def state(
         choice_sizes[col] = 0
 
     school_location_settings = """
-        SIMULATE_CHOOSER_COLUMNS:
-            - home_zone_id
-            - school_segment
-            - household_id
-            - is_student
-            - age_0_to_5
-            - age_6_to_12
-            - pemploy
         CHOOSER_ORIG_COL_NAME: home_zone_id
         ALT_DEST_COL_NAME: alt_dest
         SAMPLE_SIZE: 30