Tenure fixes

baus.py

@@ -204,7 +204,8 @@ def get_simulation_models(SCENARIO):
         "travel_model_output",
         # "travel_model_2_output",
         "hazards_slr_summary",
-        "hazards_eq_summary"
+        "hazards_eq_summary",
+        "save_tenure_indicators"
 
     ]
 
@@ -265,8 +266,6 @@ def run_models(MODE, SCENARIO):
                 "rrh_simulate",    # residential rental hedonic for units
                 "nrh_simulate",
 
-                # (based on higher of predicted price or rent)
-                "assign_tenure_to_new_units",
 
                 # uses conditional probabilities
                 "households_relocation",
@@ -275,6 +274,9 @@ def run_models(MODE, SCENARIO):
                 "reconcile_unplaced_households",
                 "jobs_transition",
 
+                # (based on higher of predicted price or rent)
+                "assign_tenure_to_new_units",
+
                 # allocate owners to vacant owner-occupied units
                 "hlcm_owner_simulate",
                 # allocate renters to vacant rental units
@@ -296,7 +298,8 @@ def run_models(MODE, SCENARIO):
                 "hazards_slr_summary",
                 "hazards_eq_summary",
                 "diagnostic_output",
-                "config"
+                "config",
+                "save_tenure_indicators"
 
             ], iter_vars=[IN_YEAR])
 

baus/models.py

@@ -119,7 +119,7 @@ def _proportional_jobs_model(
     # city but not locations to put them in.  we need to drop this demand
     drop = need_more_jobs.index.difference(locations_series.unique())
     print "We don't have any locations for these locations:\n", drop
-    need_more_jobs = need_more_jobs.drop(drop)
+    need_more_jobs = need_more_jobs.drop(drop).astype('int')
 
     # choose random locations within jurises to match need_more_jobs totals
     choices = groupby_random_choice(locations_series, need_more_jobs,
@@ -523,7 +523,7 @@ def residential_developer(feasibility, households, buildings, parcels, year,
             year=year,
             form_to_btype_callback=form_to_btype_func,
             add_more_columns_callback=add_extra_columns_func,
-            num_units_to_build=target,
+            num_units_to_build=int(target),
             profit_to_prob_func=subsidies.profit_to_prob_func,
             **kwargs)
 
@@ -731,7 +731,7 @@ def office_developer(feasibility, jobs, buildings, parcels, year,
                 form_to_btype_callback=form_to_btype_func,
                 add_more_columns_callback=add_extra_columns_func,
                 residential=False,
-                num_units_to_build=target,
+                num_units_to_build=int(target),
                 profit_to_prob_func=subsidies.profit_to_prob_func,
                 **dev_settings['kwargs'])
 

baus/summaries.py

@@ -987,7 +987,7 @@ def count_acres_with_mask(mask):
     taz_df["resacre"] = scaled_resacre(
         base_year_summary_taz.RESACRE_UNWEIGHTED, taz_df.resacre_unweighted)
     rc = regional_controls.to_frame()
-    taz_df = add_population(taz_df, year, rc)
+    taz_df = add_population(taz_df, year, rc, check_close=False)
     taz_df.totpop = taz_df.hhpop + taz_df.gqpop
     taz_df = add_employment(taz_df, year, rc)
     taz_df = add_age_categories(taz_df, year, rc)
@@ -1389,14 +1389,16 @@ def zone_forecast_inputs():
                        index_col="zone_id")
 
 
-def add_population(df, year, regional_controls):
+def add_population(df, year, regional_controls, check_close=True):
     rc = regional_controls
     target = rc.totpop.loc[year] - df.gqpop.sum()
-
     zfi = zone_forecast_inputs()
     s = df.tothh * zfi.meanhhsize
 
-    s = scale_by_target(s, target, .15)
+    if check_close:
+        s = scale_by_target(s, target, .15)
+    else:
+        s = scale_by_target(s, target)
 
     df["hhpop"] = round_series_match_target(s, target, 0)
     df["hhpop"] = df.hhpop.fillna(0)

baus/ual.py

@@ -518,7 +518,7 @@ def initialize_new_units(buildings, residential_units):
 
 
 @orca.step()
-def assign_tenure_to_new_units(residential_units, settings):
+def assign_tenure_to_new_units(residential_units, households, settings):
     """
     This data maintenance step assigns tenure to new residential units.
     Tenure is determined by comparing the fitted sale price and fitted
@@ -552,7 +552,8 @@ def assign_tenure_to_new_units(residential_units, settings):
             ColumnSpec('unit_residential_rent', min=0))))
     '''
 
-    cols = ['tenure', 'unit_residential_price', 'unit_residential_rent']
+    cols = ['tenure', 'unit_residential_price', 'unit_residential_rent',
+            'vacant_units']
     units = residential_units.to_frame(cols)
 
     # Filter for units that are missing a tenure assignment
@@ -567,6 +568,7 @@ def assign_tenure_to_new_units(residential_units, settings):
     rental_units = (units.unit_residential_rent > units.unit_residential_price)
     units.loc[~rental_units, 'tenure'] = 'own'
     units.loc[rental_units, 'tenure'] = 'rent'
+    units = unplaced_adjustment(households, units)
 
     print "Adding tenure assignment to %d new residential units" % len(units)
     print units.describe()
@@ -575,6 +577,49 @@ def assign_tenure_to_new_units(residential_units, settings):
         'tenure', units.tenure, cast=True)
 
 
+def unplaced_adjustment(households, units):
+
+    """
+    Modifies tenure assignment to new units, so that it is not only based on
+    the highest value between buying price and rent (converted to equivalent),
+    but also considers the minimum number of units required by tenure category
+    to meet demand
+
+    Data expectations
+    -----------------
+    - households: Orca table of households
+    - units: Pandas DataFrame with initial tenure assignment
+
+    Results
+    -------
+    - units: DataFrame with adjusted tenure values
+    """
+
+    hh = households.to_frame(['unit_id', 'building_id', 'tenure'])
+    vacant_units = units[units['vacant_units'] > 0]
+    min_new = {}
+    for tenure in ['own', 'rent']:
+        units_tenure = vacant_units[vacant_units['tenure'] == tenure]
+        vacant_units_tenure = units_tenure.vacant_units.sum()
+        unplaced_hh = hh[(hh['tenure'] == tenure) & (hh['unit_id'] == -1)]
+        unplaced_hh = len(unplaced_hh.index)
+        min_new[tenure] = max(unplaced_hh - vacant_units_tenure, 0)
+    complement = {'own': 'rent', 'rent': 'own'}
+    price = {'own': 'unit_residential_price', 'rent': 'unit_residential_rent'}
+    for tenure in ['own', 'rent']:
+        units_tenure = vacant_units[vacant_units['tenure'] == tenure]
+        units_comp = vacant_units[vacant_units['tenure'] == complement[tenure]]
+        if (min_new[complement[tenure]] < len(units_comp.index)) & \
+                (min_new[tenure] > len(units_tenure.index)):
+            available_units = len(units_tenure.index)
+            missing_units = min_new[tenure] - available_units
+            extra_units = len(units_comp.index) - min_new[complement[tenure]]
+            extra_units = int(min(missing_units, extra_units))
+            extra_units = units_comp.nlargest(extra_units, price[tenure])
+            units.loc[extra_units.index, 'tenure'] = tenure
+    return units
+
+
 @orca.step()
 def save_intermediate_tables(households, buildings, parcels,
                              jobs, zones, year):
@@ -768,8 +813,11 @@ def hlcm_owner_simulate(households, residential_units,
     # alternatives, for supply/demand equilibration, and needs to NOT be in the
     # choosers table, to avoid conflicting when the tables are joined
 
-    return hlcm_simulate(households, residential_units, aggregations,
-                         settings, hlcm_owner_config, 'price_equilibration')
+    correct_alternative_filters_sample(residential_units, households, 'own')
+    hlcm_simulate(orca.get_table('own_hh'), orca.get_table('own_units'),
+                  aggregations, settings, hlcm_owner_config,
+                  'price_equilibration')
+    update_unit_ids(households, 'own')
 
 
 @orca.step()
@@ -785,8 +833,68 @@ def hlcm_owner_lowincome_simulate(households, residential_units,
 @orca.step()
 def hlcm_renter_simulate(households, residential_units, aggregations,
                          settings, hlcm_renter_config):
-    return hlcm_simulate(households, residential_units, aggregations,
-                         settings, hlcm_renter_config, 'rent_equilibration')
+
+    correct_alternative_filters_sample(residential_units, households, 'rent')
+    hlcm_simulate(orca.get_table('rent_hh'), orca.get_table('rent_units'),
+                  aggregations, settings, hlcm_renter_config,
+                  'rent_equilibration')
+    update_unit_ids(households, 'rent')
+
+
+def correct_alternative_filters_sample(residential_units, households, tenure):
+    """
+    Creates modified versions of the alternatives and choosers Orca tables
+    (residential units and households), so that the parameters that will be
+    given to the hlcm_simulate() method are already filtered with the
+    alternative filters defined in the hlcm_owner and hlcm_renter yaml files.
+
+    Data expectations
+    -----------------
+    - residential_units: Orca table of residential units
+    - households: Orca table of households
+    - tenure: String for tenure. Takes the values of 'rent' or 'own'
+
+    Results
+    -------
+    None. New tables of residential units and households by tenure segment
+    are registered in Orca.
+    """
+
+    units = residential_units.to_frame()
+    units_tenure = units[units.tenure == tenure]
+    units_name = tenure + '_units'
+    orca.add_table(units_name, units_tenure, cache=True, cache_scope='step')
+    hh = households.to_frame()
+    hh_tenure = hh[hh.tenure == tenure]
+    hh_name = tenure + '_hh'
+    orca.add_table(hh_name, hh_tenure, cache=True, cache_scope='step')
+    orca.broadcast('buildings', units_name,
+                   cast_index=True, onto_on='building_id')
+    orca.broadcast(units_name, hh_name, cast_index=True, onto_on='unit_id')
+
+
+def update_unit_ids(households, tenure):
+    """
+    After running the hlcm simulation for a given tenure (own or rent), this
+    function retrieves the new unit_id values from the own_hh or rent_hh
+    Orca tables as applicable. It then updates the general households table
+    with the new unit ids that were selected by unplaced households.
+
+    Data expectations
+    -----------------
+    - households: Orca table of households
+    - tenure: String for tenure. Takes the values of 'rent' or 'own'
+
+    Results
+    -------
+    None. unit_id column gets updated in the households table.
+    """
+
+    unit_ids = households.to_frame(['unit_id'])
+    updated = orca.get_table(tenure+'_hh').to_frame(['unit_id'])
+    unit_ids.loc[unit_ids.index.isin(updated.index),
+                 'unit_id'] = updated['unit_id']
+    households.update_col_from_series('unit_id', unit_ids.unit_id, cast=True)
 
 
 @orca.step()
@@ -886,3 +994,43 @@ def balance_rental_and_ownership_hedonics(households, settings,
         utilization_ratio
 
     print "New cap rate = %.2f" % settings["cap_rate"]
+
+
+@orca.step()
+def save_tenure_indicators(households, buildings, residential_units, year):
+    """
+    Saves the tenure for households and residential units at the end of each
+    year using the "households_tenure_track" and "units_tenure_track" Orca
+    tables, so that these values can be used for any required tenure analysis.
+
+    Data expectations
+    -----------------
+    - households: Orca table of households
+    - buildings: Orca table of buildings
+    - residential_units: Orca table of residential units
+    - year: Orca injectable for current simulation year
+
+    Results
+    -------
+    - None. "households_tenure_track" and "units_tenure_track" Orca tables get
+    added for the base year, and updated for all subsequent years
+    """
+
+    households_tenure = households.to_frame(['building_id', 'unit_id',
+                                             'tenure'])
+    units_tenure = residential_units.to_frame(['unit_id', 'tenure',
+                                               'building_id'])
+    households_tenure['year'] = year
+    units_tenure['year'] = year
+    if not ('households_tenure_track' in orca.list_injectables()):
+        orca.add_injectable('households_tenure_track', households_tenure)
+        orca.add_injectable('units_tenure_track', units_tenure)
+    else:
+        households_tenure_track = \
+            orca.get_injectable('households_tenure_track')
+        units_tenure_track = orca.get_injectable('units_tenure_track')
+        households_tenure_track = \
+            households_tenure_track.append(households_tenure)
+        units_tenure_track = units_tenure_track.append(units_tenure)
+        orca.add_injectable('households_tenure_track', households_tenure_track)
+        orca.add_injectable('units_tenure_track', units_tenure_track)

baus/utils.py

@@ -207,6 +207,11 @@ def constrained_normalization(marginals, constraint, total):
 def simple_ipf(seed_matrix, col_marginals, row_marginals, tolerance=1, cnt=0):
     assert np.absolute(row_marginals.sum() - col_marginals.sum()) < 5.0
 
+    # most numpy/pandas combinations will perform this conversion
+    # automatically, but explicit is safer - see PR #98
+    if isinstance(col_marginals, pd.Series):
+        col_marginals = col_marginals.values
+
     # first normalize on columns
     ratios = col_marginals / seed_matrix.sum(axis=0)
     seed_matrix *= ratios

configs/hlcm_owner.yaml

@@ -10,8 +10,9 @@ choosers_predict_filters: (tenure == 'own')
 
 alts_fit_filters: null
 
+alts_predict_filters: (tenure == 'own')
 #alts_predict_filters: (tenure == 'own' & deed_restricted == False)
-alts_predict_filters: null
+#alts_predict_filters: null
 
 interaction_predict_filters: null
 

configs/hlcm_renter.yaml

@@ -10,8 +10,9 @@ choosers_predict_filters: (tenure == 'rent')
 
 alts_fit_filters: null
 
+alts_predict_filters: (tenure == 'rent')
 #alts_predict_filters: (tenure == 'rent' & deed_restricted == False)
-alts_predict_filters: null
+#alts_predict_filters: null
 
 interaction_predict_filters: null