Merge pull request #618 from HSLdevcom/feat/results-refactor

Major results refactor
All results are now based on events and do pollute the main loop anymore. Fully independence is not guaranteed so check that in case you want to parallelize results handling.

Author: hsl-petrhaj

.github/workflows/pythonapp.yml

@@ -1,14 +1,53 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Python application
+on:
+  push:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
 
-on: [push, pull_request]
-  
 jobs:
-  build:
+  # This job figures out whether the ref has an associated draft PR
+  context:
+    runs-on: ubuntu-latest
+    outputs:
+      isDraftOnPush: ${{ steps.detect.outputs.is_draft_on_push }}
+    steps:
+      - name: Detect draft PR for this push (if any)
+        id: detect
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const eventName = context.eventName;
+            // Default: not a draft
+            let isDraftOnPush = false;
 
+            if (eventName === 'push') {
+              // Branch name like "feature/foo" (strip "refs/heads/")
+              const branch = context.ref.replace(/^refs\/heads\//, '');
+              // Find open PRs from this branch in this repo
+              const { data: prs } = await github.rest.pulls.list({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                state: 'open',
+                head: `${context.repo.owner}:${branch}`
+              });
+              // If there is an open PR and it's draft, mark it
+              if (prs.length > 0 && prs[0].draft) {
+                isDraftOnPush = true;
+              }
+            }
+            core.setOutput('is_draft_on_push', String(isDraftOnPush));
+  
+  build:
+    needs: context
     runs-on: windows-latest
+    # Skip rules:
+    # - If event is pull_request: skip when draft
+    # - If event is push: skip when a corresponding open PR exists AND it's draft
+    if: >
+      (github.event_name == 'pull_request' && !github.event.pull_request.draft)
+      || (github.event_name == 'push' && needs.context.outputs.isDraftOnPush != 'true')
 
     steps:
     - uses: actions/checkout@v2

Scripts/assignment/departure_time.py

@@ -27,9 +27,9 @@ def __init__(self,
         self.time_periods = time_periods
         self.demand: Optional[Union[int,Dict[str,Dict[str,numpy.ndarray]]]] = None
         self.old_car_demand: Union[int,numpy.ndarray] = 0
-        self.init_demand()
+        self.init_demand_and_get_gaps()
 
-    def init_demand(self) -> Dict[str,float]:
+    def init_demand_and_get_gaps(self) -> Dict[str,float]:
         """Initialize/reset demand for all time periods.
 
         Includes all transport classes, each being set to zero.

Scripts/assignment/emme_assignment.py

@@ -164,21 +164,21 @@ def aggregate_results(self, resultdata: ResultsData):
                 if res != "transit_volumes":
                     self._node_24h(
                         transit_class, param.segment_results[res])
-        ass_classes = list(param.emme_matrices) + ["bus", "aux_transit"]
-        ass_classes.remove("walk")
-        for ass_class in ass_classes:
+        self.res_ass_classes = list(param.emme_matrices) + ["bus", "aux_transit"] #result assignment classes
+        self.res_ass_classes.remove("walk")
+        for ass_class in self.res_ass_classes:
             self._link_24h(ass_class)
 
         # Aggregate and print vehicle kms and link lengths
-        kms = dict.fromkeys(ass_classes, 0.0)
+        kms = dict.fromkeys(self.res_ass_classes, 0.0)
         vdfs = {param.roadclasses[linktype].volume_delay_func
             for linktype in param.roadclasses}
         vdfs.add(0) # Links with car traffic prohibited
         vdf_kms = {ass_class: pandas.Series(0.0, vdfs)
-            for ass_class in ass_classes}
+            for ass_class in self.res_ass_classes}
         areas = zone_param.area_aggregation
         area_kms = {ass_class: pandas.Series(0.0, areas)
-            for ass_class in ass_classes}
+            for ass_class in self.res_ass_classes}
         vdf_area_kms = {vdf: pandas.Series(0.0, areas) for vdf in vdfs}
         #The following line only works well in Python 3.7+
         linktypes = list(dict.fromkeys(param.roadtypes.values())) + list(dict.fromkeys(param.railtypes.values()))
@@ -194,7 +194,7 @@ def aggregate_results(self, resultdata: ResultsData):
             else:
                 vdf = 0
             area = belongs_to_area(link.i_node)
-            for ass_class in ass_classes:
+            for ass_class in self.res_ass_classes:
                 veh_kms = link[self._extra(ass_class)] * link.length
                 kms[ass_class] += veh_kms
                 if vdf in vdfs:
@@ -209,55 +209,23 @@ def aggregate_results(self, resultdata: ResultsData):
                 linklengths[param.railtypes[linktype]] += link.length
             else:
                 linklengths[param.roadtypes[vdf]] += link.length / 2
-
-        self._event_handler.on_daily_results_aggregated(self, network)
+        
+        network_aggregations = {"kms": kms, 
+                                "vdfs": vdfs,
+                                "areas": areas,
+                                "linktypes": linktypes,
+                                "linklengths": linklengths,
+                                "veh_kms": veh_kms,
+                                "vdf_kms": vdf_kms,
+                                "area_kms": area_kms,
+                                "vdf_area_kms": vdf_area_kms}
+
+        self._event_handler.on_daily_results_aggregated(self, network, network_aggregations)
 
         if faulty_kela_code_nodes:
             s = "Municipality KELA code not found for nodes: " + ", ".join(
                 faulty_kela_code_nodes)
             log.warn(s)
-        resultdata.print_line("\nVehicle kilometres", "result_summary")
-        for ass_class in ass_classes:
-            resultdata.print_line(
-                "{}:\t{:1.0f}".format(ass_class, kms[ass_class]),
-                "result_summary")
-            resultdata.print_data(
-                vdf_kms[ass_class], "vehicle_kms_vdfs.txt", ass_class)
-            resultdata.print_data(
-                area_kms[ass_class], "vehicle_kms_areas.txt", ass_class)
-        for vdf in vdf_area_kms:
-            resultdata.print_data(
-                vdf_area_kms[vdf], "vehicle_kms_vdfs_areas.txt", vdf)
-        resultdata.print_data(linklengths, "link_lengths.txt", "length")
-
-        # Aggregate and print numbers of stations
-        stations = pandas.Series(0, param.station_ids)
-        for node in network.regular_nodes():
-            for mode in param.station_ids:
-                if (node.data2 == param.station_ids[mode]
-                        and node[self._extra("transit_won_boa")] > 0):
-                    stations[mode] += 1
-                    break
-        resultdata.print_data(stations, "transit_stations.txt", "number")
-
-        # Aggregate and print transit vehicle kms
-        transit_modes = [veh.description for veh in network.transit_vehicles()]
-        dists = pandas.Series(0.0, transit_modes)
-        times = pandas.Series(0.0, transit_modes)
-        for ap in self.assignment_periods:
-            network = ap.emme_scenario.get_network()
-            volume_factor = param.volume_factors["bus"][ap.name]
-            for line in network.transit_lines():
-                mode = line.vehicle.description
-                headway = line[ap.extra("hw")]
-                if 0 < headway < 900:
-                    departures = volume_factor * 60/headway
-                    for segment in line.segments():
-                        dists[mode] += departures * segment.link.length
-                        times[mode] += (departures
-                                        * segment[ap.extra("base_timtr")])
-        resultdata.print_data(dists, "transit_kms.txt", "dist")
-        resultdata.print_data(times, "transit_kms.txt", "time")
 
     def calc_transit_cost(self, 
                           fares: TransitFareZoneSpecification, 
@@ -403,9 +371,9 @@ def _create_attributes(self,
             (e.g., self._extra)
         """
         # Create link attributes
-        ass_classes = list(param.emme_matrices) + ["bus"]
-        ass_classes.remove("walk")
-        for ass_class in ass_classes:
+        self.emme_ass_classes = list(param.emme_matrices) + ["bus"]
+        self.emme_ass_classes.remove("walk")
+        for ass_class in self.emme_ass_classes:
             self.emme_project.create_extra_attribute(
                 "LINK", extra(ass_class), ass_class + " volume",
                 overwrite=True, scenario=scenario)

Scripts/datahandling/zonedata.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 from pathlib import Path
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Dict, List, Tuple, Union, TYPE_CHECKING
 import numpy # type: ignore
 import pandas
 
-from events.event_handler import EventHandler
+if TYPE_CHECKING:
+    from events.event_handler import EventHandler
+
 import parameters.zone as param
 from utils.read_csv_file import read_csv_file
 from utils.zone_interval import ZoneIntervals, zone_interval

Scripts/datatypes/demand.py

@@ -31,6 +31,7 @@ def __init__(self,
         """
         self.purpose = purpose
         self.mode = mode
+        self.tour_matrix = matrix #need to also preserve the passenger matrix
         if mode == "car" and purpose.name in param.car_driver_share:
             self.matrix = param.car_driver_share[purpose.name] * matrix
         else:

Scripts/datatypes/purpose.py

@@ -70,25 +70,6 @@ def __init__(self,
     def zone_numbers(self):
         return self.zone_data.zone_numbers[self.bounds]
 
-    def print_data(self):
-        self.resultdata.print_data(
-            pandas.Series(
-                sum(self.generated_tours.values()), self.zone_numbers),
-            "generation.txt", self.name)
-        self.resultdata.print_data(
-            pandas.Series(
-                sum(self.attracted_tours.values()),
-                self.zone_data.zone_numbers),
-            "attraction.txt", self.name)
-        demsums = {mode: self.generated_tours[mode].sum()
-            for mode in self.modes}
-        demand_all = float(sum(demsums.values()))
-        mode_shares = {mode: demsums[mode] / demand_all for mode in demsums}
-        self.resultdata.print_data(
-            pandas.Series(mode_shares),
-            "mode_share.txt", self.name)
-
-
 class TourPurpose(Purpose):
     """Standard two-way tour purpose.
 
@@ -135,19 +116,6 @@ def __init__(self, specification, zone_data, resultdata):
         else:
             self.park_and_ride_model = None
 
-    def print_data(self, pnr_iteration=0):
-        Purpose.print_data(self)
-        for mode in self.histograms:
-            self.resultdata.print_data(
-                self.histograms[mode].histogram, "trip_lengths.txt",
-                "{}_{}".format(self.name, mode[0]))
-            self.resultdata.print_matrix(
-                self.aggregates[mode].matrix, "aggregated_demand",
-                "{}_{}".format(self.name, mode), pnr_iteration)
-            self.resultdata.print_data(
-                self.own_zone_aggregates[mode].array,
-                "own_zone_demand.txt", "{}_{}".format(self.name, mode[0]))
-
     def init_sums(self):
         for mode in self.modes:
             self.generated_tours[mode] = numpy.zeros_like(self.zone_numbers)
@@ -165,17 +133,11 @@ def calc_prob(self, impedance):
             Mode (car/transit/bike/walk) : dict
                 Type (time/cost/dist) : numpy 2d matrix
         """
-        def print_pnr_utility(pnr_utility: numpy.ndarray, result_path: Path):
-            # TODO: This is a temporary solution to print the park and ride utility
-            omx_file = omx.open_file(result_path / 'park_and_ride_utility.omx', 'w')
-            omx_file.create_mapping('zone_number', self.zone_data.zone_numbers)
-            omx_file['park_and_ride_utility'] = pnr_utility
-            omx_file.close()
+        #Add park and ride impedance for models that include it
         if self.park_and_ride_model is not None:
-            pnr_utility = self.park_and_ride_model.get_logsum()
-            impedance['park_and_ride'] = {'utility': pnr_utility,
+            self.pnr_utility = self.park_and_ride_model.get_logsum()
+            impedance['park_and_ride'] = {'utility': self.pnr_utility,
                                           'dist': impedance['car']['dist']}
-            print_pnr_utility(pnr_utility, Path(self.resultdata.path))
         self.prob = self.model.calc_prob(impedance)
         self.dist = impedance["car"]["dist"]
 
@@ -218,9 +180,6 @@ def calc_pnr_demand(self, population, estimation_mode=False, log=None):
                               self.zone_data.zone_index(tour.dest)] += 1
         log.info(f"Matrix contains {od_matrix.sum()} park and ride tours")
         demand = {}
-        # if estimation_mode:
-        #     omx_file = omx.open_file(f"{self.resultdata.path}/estimation/demand_{self.name}.omx","w")
-        #     omx_file.create_mapping("zone_number",self.zone_data.all_zone_numbers)
 
         car_demand, transit_demand = self.park_and_ride_model.distribute_demand(od_matrix)
         pnr_purpose = ParkAndRidePseudoPurpose(self)
@@ -240,38 +199,28 @@ def calc_demand(self, estimation_mode=False, add_sec_dest: bool = True, pnr_iter
         """
         tours = self.gen_model.get_tours()
         demand = {}
-        if estimation_mode:
-            omx_file = omx.open_file(f"{self.resultdata.path}/estimation/demand_{self.name}.omx","w")
-            omx_file.create_mapping("zone_number",self.zone_data.all_zone_numbers)
         for mode in self.modes:
             mtx = (self.prob.pop(mode) * tours).T
+            self.orig_purpose_demand = mtx
             if mode == "park_and_ride":
                 car_demand, transit_demand = self.park_and_ride_model.distribute_demand(mtx)
                 pnr_purpose = ParkAndRidePseudoPurpose(self)
                 demand["pnr_car"] = Demand(pnr_purpose, "car", car_demand)
                 demand["pnr_transit"] = Demand(pnr_purpose, "transit", transit_demand)
-                # if True: 
-                #     omx_file["pnr_car"] = car_demand
-                #     omx_file["pnr_transit"] = transit_demand
             else:
                 if add_sec_dest:
                     try:
                         self.sec_dest_purpose.gen_model.add_tours(mtx, mode, self)
                     except AttributeError: #If the tour does not generate sec_dest tours
                         pass
                 demand[mode] = Demand(self, mode, mtx)
-                if estimation_mode:
-                    omx_file[mode] = mtx
             self.attracted_tours[mode] = mtx.sum(0)
             self.generated_tours[mode] = mtx.sum(1)
             self.histograms[mode].count_tour_dists(mtx, self.dist)
             self.aggregates[mode].aggregate(pandas.DataFrame(
                 mtx, self.zone_numbers, self.zone_data.zone_numbers))
             self.own_zone_aggregates[mode].aggregate(pandas.Series(
                 numpy.diag(mtx), self.zone_numbers))
-        self.print_data(pnr_iteration)
-        if estimation_mode:
-            omx_file.close()
         return demand
 
 
@@ -380,11 +329,4 @@ def calc_prob(self, mode, impedance, orig, dests):
             dest_imp[mtx_type] = (impedance[mtx_type][dests, :]
                                   + impedance[mtx_type][:, orig]
                                   - impedance[mtx_type][dests, orig][:, numpy.newaxis])
-        return self.model.calc_prob(mode, dest_imp, orig, dests)
-
-    def print_data(self):
-        self.resultdata.print_data(
-            pandas.Series(
-                sum(self.attracted_tours.values()),
-                self.zone_data.zone_numbers),
-            "attraction.txt", self.name)
+        return self.model.calc_prob(mode, dest_imp, orig, dests)

Scripts/datatypes/tour.py

@@ -143,7 +143,9 @@ def choose_mode(self, is_car_user: bool):
 
     @property
     def sustainable_access(self):
-        return -self.purpose.sustainable_access[self.orig]
+        if self.orig in self.purpose.sustainable_access:
+            return (-1)*self.purpose.sustainable_access[self.orig]
+        return -1 #Fallback
 
     def choose_destination(self, sec_dest_tours: Dict[str, Dict[int, Dict[int, List['Tour']]]]):
         """Choose primary destination for the tour.

Scripts/demand/external.py Scripts/demand/all_external.pyScripts/demand/external.py renamed to Scripts/demand/all_external.py

@@ -164,8 +164,7 @@ def generate_tours(self):
                     self.purpose_dict[purpose].gen_model.tours += nr_tours
                 nr_tours_sums["-".join(combination)] = nr_tours.sum()
             result_data[age] = nr_tours_sums.sort_index()
-        self.resultdata.print_matrix(
-            result_data, "tour_combinations", "tour_combinations")
+        self.tour_generation_model.result = result_data
 
     def generate_tour_probs(self) -> Dict[Tuple[int,int], numpy.ndarray]:
         """Generate matrices of cumulative tour combination probabilities.