Merge pull request berkeleyflow#36 from berkeleyflow/grid_fix

Grid fix

Author: AboudyKreidieh

benchmarks/grid0.py

@@ -70,6 +70,7 @@ def get_non_flow_params(enter_speed, additional_net_params):
 num_cars_right = 1
 num_cars_top = 1
 num_cars_bot = 1
+edge_inflow = 300
 rl_veh = 0
 tot_cars = (num_cars_left + num_cars_right) * m \
            + (num_cars_bot + num_cars_top) * n
@@ -81,7 +82,8 @@ def get_non_flow_params(enter_speed, additional_net_params):
               "rl_veh": rl_veh}
 
 additional_env_params = {"target_velocity": 50, "num_steps": HORIZON,
-                         "control-length": 150, "switch_time": 3.0}
+                         "control-length": 150, "switch_time": 2.0,
+                         "total_inflow": n*m*edge_inflow}
 
 additional_net_params = {"speed_limit": 35, "grid_array": grid_array,
                          "horizontal_lanes": 1, "vertical_lanes": 1}
@@ -98,7 +100,7 @@ def get_non_flow_params(enter_speed, additional_net_params):
              speed_mode="right_of_way")
 
 initial_config, net_params = \
-    get_flow_params(v_enter, 300, n, m, additional_net_params)
+    get_flow_params(v_enter, edge_inflow, n, m, additional_net_params)
 
 
 flow_params = dict(

benchmarks/grid1.py

@@ -70,6 +70,7 @@ def get_non_flow_params(enter_speed, additional_net_params):
 num_cars_right = 1
 num_cars_top = 1
 num_cars_bot = 1
+edge_inflow = 600
 rl_veh = 0
 tot_cars = (num_cars_left + num_cars_right) * m \
            + (num_cars_bot + num_cars_top) * n
@@ -81,7 +82,8 @@ def get_non_flow_params(enter_speed, additional_net_params):
               "rl_veh": rl_veh}
 
 additional_env_params = {"target_velocity": 50, "num_steps": HORIZON,
-                         "control-length": 150, "switch_time": 3.0}
+                         "control-length": 150, "switch_time": 2.0,
+                         "total_inflow": edge_inflow*n*m}
 
 additional_net_params = {"speed_limit": 35, "grid_array": grid_array,
                          "horizontal_lanes": 1, "vertical_lanes": 1}
@@ -98,7 +100,7 @@ def get_non_flow_params(enter_speed, additional_net_params):
              speed_mode="right_of_way")
 
 initial_config, net_params = \
-    get_flow_params(v_enter, 600, n, m, additional_net_params)
+    get_flow_params(v_enter, edge_inflow, n, m, additional_net_params)
 
 
 flow_params = dict(

flow/core/rewards.py

@@ -136,9 +136,8 @@ def penalize_tl_changes(env, actions, gain=1):
     :param gain: {float} - multiplicative factor on the action penalty
     :return: a penalty on vehicle delays and traffic light switches
     """
-    delay = min_delay(env)
-    action_penalty = gain * np.sum(actions)
-    return delay - action_penalty
+    action_penalty = gain * np.sum(np.round(actions))
+    return -action_penalty
 
 
 def penalize_headway_variance(vehicles, vids, normalization=1, penalty_gain=1,

flow/envs/green_wave_env.py

@@ -376,6 +376,7 @@ class PO_TrafficLightGridEnv(TrafficLightGridEnv):
     def __init__(self, env_params, sumo_params, scenario):
         super().__init__(env_params, sumo_params, scenario)
         self.num_observed = self.grid_array.get("num_observed", 2)
+        self.total_inflow = env_params.additional_params["total_inflow"]
         self.observed_ids = []
 
     @property
@@ -453,7 +454,11 @@ def get_po_state(self):
                                         self.last_change.flatten().tolist()]))
 
     def compute_reward(self, state, rl_actions, **kwargs):
-        return rewards.min_delay(self)
+        hour_frac = self.horizon*self.sim_step/3600
+        delay_reward = rewards.min_delay(self)/(self.total_inflow*hour_frac)
+        switch_penalty = rewards.penalize_tl_changes(rl_actions, gain=0.2)
+        switch_penalty_norm = switch_penalty/(self.rows*self.cols)
+        return delay_reward + switch_penalty_norm
 
     def additional_command(self):
         # specify observed vehicles