Count every trip (the "Error bars" branch)

Error_bars/Public_Dashboard/add_new_label_fields.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+
+# This script adds the following fields to confirmed trips in Stage_analysis_timeseries:
+# label_assist_labels, sensed_mode, and algorithm_chosen.
+
+################ IMPORTANT
+# Make sure you are in an emission environment.
+# Make sure that when you run this script, you are in a folder that has the conf folder from e-mission-server.
+# Otherwise, e-mission-server/emission/core/get_database.py can't find conf/storage/db.conf.sample
+# If you need it, you can copy the conf folder recursively to your current directory with: 
+# cp -r $EMISSION_SERVER_HOME/conf .
+
+import sys
+import time
+
+# My path looked like this: /Users/mallen2/OpenPATH_Data/e-mission-server
+
+sys.path.insert(0, input("Enter your path to the emission server: ") )   # maybe I could try an emission import and then do this as a catch?
+import emission.storage.decorations.trip_queries as esdt
+import emission.storage.timeseries.builtin_timeseries as estbt
+import emission.core.wrapper.entry as ecwe
+
+# Get the Stage_analysis_timeseries collection
+import emission.core.get_database as edb
+Stage_analysis_timeseries = edb.get_analysis_timeseries_db()
+
+# Get all confirmed trips
+confirmed_trips = [doc for doc in Stage_analysis_timeseries.find({"metadata.key":"analysis/confirmed_trip"})]
+
+def get_label_assist_confidences(ct):
+    inference = ct['data']["inferred_labels"]
+    confidences = {}
+    for label_type in LABEL_CATEGORIES:
+        counter = {}
+        for line in inference:
+            if label_type not in line["labels"]: continue  # Seems we have some incomplete tuples!
+            val = line["labels"][label_type]
+            if val not in counter: counter[val] = 0
+            counter[val] += line["p"]
+
+        if len(counter) > 0:
+            confidences[label_type] = counter
+    return confidences
+
+def get_sensed_mode_fractions(ct):
+
+    # The commented out labels are those seen in kennykos's mobilitynet-analysis-scripts classification_analysis notebook.   
+    sensed_mode_types = {
+        0 : 'UNKNOWN',    # UNKNOWN
+        1 : 'WALKING',    # WALKING
+        2 : 'CYCLING',    # BICYCLING
+        3 : 'BUS',        # BUS
+        4 : 'TRAIN',      # TRAIN
+        5 : 'CAR',        # CAR
+        6 : 'AIR_OR_HSR', # AIR_OR_HSR
+        7 : 'SUBWAY',      # SUBWAY
+        8 : 'TRAIN',      # TRAM
+        9 : 'TRAIN',      # LIGHT_RAIL
+    }
+    # These keys were found in emission/core/wrapper/modeprediction.py:
+    #{0: "unknown", 1: "walking",2: "bicycling",
+    #3: "bus", 4: "train", 5: "car", 6: "air_or_hsr",
+    #7: "subway", 8: "tram", 9: "light_rail"}
+
+    # Mapping for GIS based mode detection, with labels that correspond to those in the default dictionary to match with energy_intensity.csv
+
+        #0 : "Not a trip",               #'UNKNOWN',    # UNKNOWN   # We could alternatively assume a middle ground mode.
+        #1 : "Walk",                     #'WALKING',    # WALKING
+        #2 : "Bike",                     #'CYCLING',    # BICYCLING
+        #3 : "Bus",                      #'BUS',        # BUS
+        #4 : "Train",                    #'TRAIN',      # TRAIN
+        #5 : "Car, drove alone",         #'CAR',        # CAR
+        #6 : "air",                      #'AIR_OR_HSR', # AIR_OR_HSR
+        #7 : "Train",                    #'SUBWAY',      # SUBWAY
+        #8 : "Train",                    #'TRAIN',      # TRAM
+        #9 : "Train"                     #'TRAIN',      # LIGHT_RAIL
+
+    # Get the segments for the trip.
+    #cleaned_section will only have walk/bike/automotive, inferred_section is the one that has bus/train/car etc 
+    segments = esdt.get_sections_for_trip(key = "analysis/inferred_section", user_id = ct["user_id"], trip_id = ct['data']['cleaned_trip'])
+
+    # get pairs of mode type and duration
+    trip_mode_durations = {}
+    total_dur = 0
+    for s in segments:
+        # the sensed mode is a number in the database, so I'm relabeling it as a string.
+        mode = sensed_mode_types[s['data']['sensed_mode']]
+        duration = s['data']['duration']
+
+        if mode not in trip_mode_durations.keys(): trip_mode_durations[mode] = 0
+        trip_mode_durations[mode] += duration
+
+        total_dur += duration
+    # convert the durations to fractions of the total segment moving time (not the trip time, since trips include stop times)
+    return {mode: duration/total_dur  for mode,duration in trip_mode_durations.items()}
+
+LABEL_CATEGORIES = ['mode_confirm','purpose_confirm','replaced_mode']
+sens_count = 0
+la_count = 0
+
+# For each trip, find its segments and get consolidated label assist labels.
+# update these trips in the database
+print("Updating trip documents")
+t1 = time.time()
+for ct in confirmed_trips:
+
+    confidences = get_label_assist_confidences(ct)
+    la_mode_is_confident = 'mode_confirm' in confidences and max(list(confidences['mode_confirm'].values())) > 0.25
+
+    # Add consolidated label assist confidences to their own field
+    ct['data']['label_assist_confidences'] = confidences
+
+    # Select an algorithm
+    if 'mode_confirm' in ct['data']['user_input']:
+        ct['data']['algorithm_chosen'] = 'user_input'
+    elif la_mode_is_confident:
+        la_count += 1
+        ct['data']['algorithm_chosen'] = 'label_assist'
+    else:
+        sens_count += 1
+        ct['data']['algorithm_chosen'] = 'sensing'
+
+    # Add the mode:duration dictionary to the confirmed trip data as sensed_mode.
+    ct["data"]["sensed_mode"] = get_sensed_mode_fractions(ct)
+
+    # Update the corresponding confirmed_trip entry in the database.
+    estbt.BuiltinTimeSeries.update(ecwe.Entry(ct))
+
+print(f"Updated {len(confirmed_trips)} total trips in {time.time()-t1} seconds.")
+print(f"There are {la_count} trips where label assist is the chosen algorithm\nand {sens_count} trips where sensing is the chosen algorithm.")

Error_bars/Public_Dashboard/auxiliary_files/energy_intensity.csv

@@ -0,0 +1,17 @@
+mode,fuel,(kWH)/trip,EI(kWH/PMT),energy_intensity_factor,energy_intensity_units,CO2_factor,CO2_factor_units
+"Gas Car, drove alone",gasoline,0,,5170,BTU/PMT,157.2,lb_CO2/MMBTU
+"Gas Car, with others",gasoline,0,,2585,BTU/PMT,157.2,lb_CO2/MMBTU
+"E-car, drove alone",electric,0,0.25,0.25,kWH/PMT,1166,lb_CO2/MWH
+"E-car, with others",electric,0,0.125,0.125,kWH/PMT,1166,lb_CO2/MWH
+Taxi/Uber/Lyft,gasoline,0,,7214,BTU/PMT,157.2,lb_CO2/MMBTU
+Bus,diesel,0,,4560,BTU/PMT,161.3,lb_CO2/MMBTU
+Free Shuttle,diesel,0,,4560,BTU/PMT,161.3,lb_CO2/MMBTU
+Train,electric,0,0.37,0.37,kWH/PMT,1166,lb_CO2/MWH
+Scooter share,electric,0.0041,0.027,0.027,kWH/PMT,1166,lb_CO2/MWH
+Pilot ebike,electric,0,0.022,0.022,kWH/PMT,1166,lb_CO2/MWH
+Bikeshare,human_powered,0.09,0,0,kWH/PMT,1166,lb_CO2/MWH
+Walk,human_powered,0,,0,,0,0
+Skate board,human_powered,0,,0,,0,0
+Regular Bike,human_powered,0,,0,,0,0
+Not a Trip,none,0,,0,,0,0
+No Travel,none,0,,0,,0,0

Error_bars/Public_Dashboard/auxiliary_files/mode_labels.csv

@@ -0,0 +1,32 @@
+replaced_mode,mode_confirm,mode_clean
+drove_alone,drove_alone,"Gas Car, drove alone"
+e_car_drove_alone,e_car_drove_alone,"E-car, drove alone"
+work_vehicle,work_vehicle,"Gas Car, drove alone"
+bus,bus,Bus
+train,train,Train
+free_shuttle,free_shuttle,Free Shuttle
+"train,_bus and walk","train,_bus and walk",Train
+train_and pilot e-bike,train_and pilot e-bike,Train
+taxi,taxi,Taxi/Uber/Lyft
+friend_picked me up,friend_picked me up,"Gas Car, with others"
+carpool_w/ friend to work,carpool_w/ friend to work,"Gas Car, with others"
+friend_carpool to work,friend_carpool to work,"Gas Car, with others"
+carpool_to work,carpool_to work,"Gas Car, with others"
+friend/co_worker carpool,friend/co_worker carpool,"Gas Car, with others"
+carpool_to lunch,carpool_to lunch,"Gas Car, with others"
+carpool,carpool,"Gas Car, with others"
+carpool_for lunch,carpool_for lunch,"Gas Car, with others"
+carpool_lunch,carpool_lunch,"Gas Car, with others"
+shared_ride,shared_ride,"Gas Car, with others"
+e_car_shared_ride,e_car_shared_ride,"E-car, with others"
+bikeshare,bikeshare,Bikeshare
+scootershare,scootershare,Scooter share
+pilot_ebike,pilot_ebike,E-bike
+e-bike,e-bike,E-bike
+walk,walk,Walk
+skateboard,skateboard,Skate board
+bike,bike,Regular Bike
+the_friend who drives us to work was running errands after the shift before dropping me off. not a trip of mine.,the_friend who drives us to work was running errands after the shift before dropping me off. not a trip of mine.,Not a Trip
+not_a_trip,not_a_trip,Not a Trip
+no_travel,,No Travel
+same_mode,,Same Mode

Error_bars/Public_Dashboard/auxiliary_files/purpose_labels.csv

@@ -0,0 +1,47 @@
+purpose_confirm,bin_purpose
+work_travel,Work
+work,Work
+home,Home
+meal,Meal
+shopping,Shopping
+personal_med,Personal/Medical
+exercise,Recreation/Exercise
+transit_transfer,Transit transfer
+pick_drop,Pick-up/Drop off
+entertainment,Entertainment/Social
+car_mechanic,Other
+school,School
+revisado_bike,Other
+placas_de carro,Other
+community_walk,Entertainment/Social
+gardening,Entertainment/Social
+visiting,Entertainment/Social
+church,Religious
+community_garden,Entertainment/Social
+community_meeting,Entertainment/Social
+visit_a friend,Entertainment/Social
+aseguranza,Other
+meeting_bike,Entertainment/Social
+gas_station,Other
+iglesia,Religious
+curso,School
+mi_hija recién aliviada,Entertainment/Social
+servicio_comunitario,Entertainment/Social
+pago_de aseguranza,Other
+grupo_comunitario,Entertainment/Social
+caminata_comunitaria,Entertainment/Social
+bank,Other
+religious,Religious
+no_travel,No travel
+work_break - short walk,Entertainment/Social
+work_- lunch break,Meal
+friend_was running errands before dropping me off after work,Other
+"multiple_errands, etc.",Other
+lunch_break,Meal
+break,Entertainment/Social
+pet,Entertainment/Social
+recording_performance at park,Entertainment/Social
+not_a trip,not_a_trip
+on_the way home,Home
+other,Other
+nan,nan

Error_bars/Public_Dashboard/count_functions.py

@@ -0,0 +1,154 @@
+# Final mode functions
+def sensed_mode(mode):
+    # When selecting a mode, use the value accepted by the energy intensity dataframe
+    # Below is from dic_mode
+    '''{'drove_alone': 'Car, drove alone',
+             'work_vehicle': 'Car, drove alone',
+             'bus': 'Bus',
+             'train': 'Train',
+             'free_shuttle': 'Free Shuttle',
+             'train,_bus and walk': 'Train',
+             'train_and pilot e-bike': 'Train',
+             'taxi': 'Taxi/Uber/Lyft',
+             'friend_picked me up': 'Car, with others',
+             'carpool_w/ friend to work': 'Car, with others',
+             'friend_carpool to work': 'Car, with others',
+             'carpool_to work': 'Car, with others',
+             'friend/co_worker carpool': 'Car, with others',
+             'carpool_to lunch': 'Car, with others',
+             'carpool': 'Car, with others',
+             'carpool_for lunch': 'Car, with others',
+             'carpool_lunch': 'Car, with others',
+             'shared_ride': 'Car, with others',
+             'bikeshare': 'Bikeshare',
+             'scootershare': 'Scooter share',
+             'pilot_ebike': 'Pilot ebike',
+             'walk': 'Walk',
+             'skateboard': 'Skate board',
+             'bike': 'Regular Bike',
+             'the_friend who drives us to work was running errands after the shift before dropping me off. not a trip of mine.': 'Not a Trip',
+             'not_a_trip': 'Not a Trip',
+             'no_travel': 'No Travel',
+             'same_mode': 'Same Mode',
+             nan: 'nan'})'''
+
+    if mode == "unknown" or mode == "UNKNOWN":       # How should unknown and other be handled?
+        return "Not a trip"
+    elif mode == "walking" or mode == "WALKING": 
+        return "Walk"
+    elif mode == "Regular Bike" or mode == "CYCLING":   # expand to bike, ebike, escooter
+        return "Bike"
+    elif mode ==  "bus" or mode == "BUS": 
+        return "Bus"
+    elif mode == "train" or mode == "TRAIN":     # collapsed with subway, tram, light rail because we lack intensities for those
+        return "Train"
+    elif mode ==  "car" or mode == "CAR":      # expand to Car, drove alone and Car, with others
+        return "Car, drove alone"
+    elif mode ==  "air_or_hsr" or mode == "AIR_OR_HSR":   # keep as air for now
+        return "air"
+    elif mode == "subway" or mode == "SUBWAY": 
+        return "Train"
+    elif mode == "tram" or mode == "TRAM": 
+        return "Train"
+    elif mode == "light_rail" or mode == "LIGHT_RAIL":
+        return "Train"
+    else:
+        Warning("Sensed mode had a different label than expected")
+
+    '''sensed_mode_types = {0: "unknown", 1: "walking",2: "bicycling",
+                    3: "bus", 4: "train", 5: "car", 6: "air_or_hsr",
+                    7: "subway", 8: "tram", 9: "light_rail"}'''
+
+    '''UNKNOWN = 0 Other
+    WALKING = 1  Walk
+    BICYCLING = 2 Bike
+    BUS = 3       Bus 
+    TRAIN = 4     Train
+    CAR = 5       Drove Alone
+    AIR_OR_HSR = 6 Air
+    SUBWAY = 7     Train
+    TRAM = 8        Train
+    LIGHT_RAIL = 9  Train'''
+
+def get_final_mode(trip):
+
+    if not trip['Mode_confirm'] == 'nan':
+        return trip["Mode_confirm"]
+
+    if  trip["algorithm_chosen"] == "sensing" and len(trip["sensed_mode"]) > 0:
+        sensed_label = max(trip["sensed_mode"], key=trip["sensed_mode"].get)
+        final_mode = sensed_mode(sensed_label)
+    else:  
+        final_mode = trip["la_mode"]
+
+    return final_mode
+
+def get_footprints_by_mode(df,footprint_col_name,mode_column):
+    '''Returns a dictionary of total footprint values for each mode. Also includes an overall total.'''
+    footprints_by_mode = {}
+    for mode in df[mode_column].unique():
+        single_mode_df = df[df[mode_column] == mode]
+        footprints_by_mode[mode] = single_mode_df[footprint_col_name].sum()
+
+    total = sum(footprints_by_mode.values())
+    footprints_by_mode["Total"] = total
+    return footprints_by_mode
+
+def get_inferred_footprint_intervals(footprint_dict,footprint_rel_errors):
+    '''
+    Calculate confidence intervals for footprint metrics
+    footprint_dict: a dictionary by mode of footprint values (carbon or energy)
+    footprint_rel_errors: a dictionary by mode containing upper and lower relative errors.\
+        it is a subdictionary in the Error Rates collection document, under the name 'carbon' or 'energy'
+    '''
+    inferred_footprint_intervals = {}
+    for mode in footprint_dict:
+        if mode == "Other": continue
+
+        estimate = footprint_dict[mode]
+        if estimate != 0:
+            lower_rel_error = footprint_rel_errors[mode][0]
+            upper_rel_error = footprint_rel_errors[mode][1]
+            footprint_interval = [estimate*(1 + lower_rel_error), estimate*(1+upper_rel_error)]
+        else:
+            footprint_interval = [0,0]
+
+        inferred_footprint_intervals[mode]= {"estimate": estimate, "lower": footprint_interval[0], "upper": footprint_interval[1]}
+
+    return inferred_footprint_intervals
+
+def make_rel_errors(max_size):
+    import numpy as np
+    upper= np.random.rand(1)*max_size
+    lower = -np.random.rand(1)*max_size
+    return  [float(lower),float(upper)]
+
+def construct_mock_errors():
+
+    import emission.core.get_database as edb
+
+    Error_Rates = edb.get_Error_Rates_db()
+    # mode count intervals, (purpose count intervals later), carbon intervals, energy intervals
+    # Maybe eventually provide the bootstrap distributions?
+    modes = ['Bikeshare','Walk', 'Regular Bike', 'Pilot ebike', 'Scooter share', 'Car, drove alone', 
+            'Car, with others', 'Taxi/Uber/Lyft', 'Bus', 'Train', 'Free Shuttle', 
+            'Air', 'Other', 'Not a Trip','Total']
+    mode_rel_errors = {}
+    carbon_rel_errors = {}
+    energy_rel_errors = {}
+    for m in modes:
+        if m is not 'Total':
+            mode_rel_errors[m] = make_rel_errors(0.15)
+        carbon_rel_errors[m] = make_rel_errors(0.2)
+        energy_rel_errors[m] = make_rel_errors(0.2)
+
+    mode_rel_errors['Not a Trip'] = make_rel_errors(0.15)
+
+    error_dictionary = {}
+    error_dictionary["counts"] = {}
+    error_dictionary["counts"]["mode_confirm"] = mode_rel_errors
+    error_dictionary["carbon"] = carbon_rel_errors
+    error_dictionary["energy"] = energy_rel_errors
+    error_dictionary["distance"] = {"interval": make_rel_errors(0.05), "mean": 0.01, "variance":0.02}
+    Error_Rates.drop()
+    Error_Rates.insert_one(error_dictionary)