diff --git a/alembic/versions/786584a43a32_added_newsflash_features.py b/alembic/versions/786584a43a32_added_newsflash_features.py new file mode 100644 index 000000000..2d6cf8c4e --- /dev/null +++ b/alembic/versions/786584a43a32_added_newsflash_features.py @@ -0,0 +1,39 @@ +"""Added newsflash features + +Revision ID: 786584a43a32 +Revises: 10023013f155 +Create Date: 2021-02-24 21:00:01.883726 + +""" + +# revision identifiers, used by Alembic. +revision = '786584a43a32' +down_revision = '10023013f155' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_unique_constraint(None, 'news_flash', ['id']) + op.create_table('newsflash_features', + sa.Column('id', sa.BigInteger(), nullable=False), + sa.Column('newsflash_id', sa.BigInteger(), nullable=True), + sa.Column('version', sa.Integer(), nullable=False), + sa.Column('timestamp', sa.DateTime(), nullable=False), + sa.Column('is_urban', sa.Boolean(), nullable=True), + sa.ForeignKeyConstraint(['newsflash_id'], ['news_flash.id'], ), + sa.PrimaryKeyConstraint('id') + ) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('newsflash_features') + op.drop_constraint(None, 'news_flash', type_='unique') + # ### end Alembic commands ### diff --git a/anyway/flask_app.py b/anyway/flask_app.py index c0e9c2d3c..52f079ef3 100755 --- a/anyway/flask_app.py +++ b/anyway/flask_app.py @@ -1592,13 +1592,14 @@ def infographics_data(): number_of_years_ago = request.values.get("years_ago", BE_CONST.DEFAULT_NUMBER_OF_YEARS_AGO) lang: str = request.values.get("lang", "he") logging.debug( - ("getting infographics data for news_flash_id: {news_flash_id}, " + - "in time period:{number_of_years_ago}, lang:{lang}").format( - news_flash_id=news_flash_id, number_of_years_ago=number_of_years_ago, - lang=lang - ) + ( + "getting infographics data for news_flash_id: {news_flash_id}, " + + "in time period:{number_of_years_ago}, lang:{lang}" + ).format(news_flash_id=news_flash_id, number_of_years_ago=number_of_years_ago, lang=lang) + ) + output = get_infographics_data( + news_flash_id=news_flash_id, years_ago=number_of_years_ago, lang=lang ) - output = get_infographics_data(news_flash_id=news_flash_id, years_ago=number_of_years_ago, lang=lang) if not output: log_bad_request(request) return abort(http_client.NOT_FOUND) diff --git a/anyway/infographics_utils.py b/anyway/infographics_utils.py index 077480598..9785de884 100755 --- a/anyway/infographics_utils.py +++ b/anyway/infographics_utils.py @@ -16,7 +16,13 @@ from sqlalchemy import desc from flask_babel import _ from anyway.backend_constants import BE_CONST -from anyway.models import NewsFlash, AccidentMarkerView, InvolvedMarkerView, VehicleMarkerView +from anyway.models import ( + NewsFlash, + AccidentMarkerView, + InvolvedMarkerView, + VehicleMarkerView, + NewsflashFeatures, +) from anyway.parsers import resolution_dict from anyway.app_and_db import db from anyway.infographics_dictionaries import ( @@ -26,9 +32,10 @@ english_accident_type_dict, segment_dictionary, english_injury_severity_dict, - hebrew_accident_severity_dict + hebrew_accident_severity_dict, ) from anyway.parsers import infographics_data_cache_updater +from anyway.rules.newsflash_feature_generator import NewsflashFeatureGenerator from anyway.utilities import parse_age_from_range from anyway.vehicle_type import VehicleCategory @@ -40,9 +47,10 @@ class RequestParams: """ news_flash_obj: NewsFlash + newsflash_features: NewsflashFeatures years_ago: int location_text: str - location_info: Dict[str, Any] + location_info: Optional[Dict[str, Any]] resolution: Dict gps: Dict start_time: datetime.date @@ -79,7 +87,7 @@ class Widget: request_params: RequestParams name: str - rank: int + rank: float items: Union[Dict, List] text: Dict meta: Optional[Dict] @@ -87,7 +95,7 @@ class Widget: def __init__(self, request_params: RequestParams, name: str): self.request_params = request_params self.name = name - self.rank = -1 + self.rank = 1.0 self.items = {} self.text = {} self.meta = None @@ -95,7 +103,7 @@ def __init__(self, request_params: RequestParams, name: str): def get_name(self) -> str: return self.name - def get_rank(self) -> int: + def get_rank(self) -> float: return self.rank # noinspection PyMethodMayBeStatic @@ -104,9 +112,9 @@ def is_in_cache(self) -> bool: return True # noinspection PyMethodMayBeStatic - def is_included(self) -> bool: + def calc_rank(self, requestParams: RequestParams) -> float: """Whether this widget is included in the response""" - return True + return self.rank def generate_items(self) -> None: """ Generates the data of the widget and set it to self.items""" @@ -134,7 +142,6 @@ def serialize(self): else: output["meta"] = {} output["meta"]["rank"] = self.rank - output["meta"]["information"] = "Placeholder: This Widget shows information of accidenents in Israel with comparison of vehicle types / locations / injured types." return output @@ -162,7 +169,6 @@ class AccidentCountBySeverityWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 1 def generate_items(self) -> None: self.items = AccidentCountBySeverityWidget.get_accident_count_by_severity( @@ -204,7 +210,6 @@ class MostSevereAccidentsTableWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 2 def generate_items(self) -> None: self.items = MostSevereAccidentsTableWidget.prepare_table( @@ -283,7 +288,6 @@ class MostSevereAccidentsWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 3 def generate_items(self) -> None: self.items = MostSevereAccidentsWidget.get_most_severe_accidents( @@ -328,7 +332,6 @@ class StreetViewWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 4 def generate_items(self) -> None: self.items = { @@ -345,7 +348,6 @@ class HeadOnCollisionsComparisonWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 5 # self.text = {"title": "תאונות קטלניות ע״פ סוג"} # self.text = {"title": "fatal accidents by type"} @@ -385,21 +387,21 @@ def get_head_to_head_stat(self) -> Dict: ) road_sums = self.sum_count_of_accident_type( - road_data, BE_CONST.AccidentType.HEAD_ON_FRONTAL_COLLISION) + road_data, BE_CONST.AccidentType.HEAD_ON_FRONTAL_COLLISION + ) all_roads_sums = self.sum_count_of_accident_type( - all_roads_data, BE_CONST.AccidentType.HEAD_ON_FRONTAL_COLLISION) - - res = {self.SPECIFIC_ROAD_SUBTITLE: [ - {"desc": "frontal", - "count": road_sums["given"] - }, - {"desc": "others", - "count": road_sums["others"]} - ], + all_roads_data, BE_CONST.AccidentType.HEAD_ON_FRONTAL_COLLISION + ) + + res = { + self.SPECIFIC_ROAD_SUBTITLE: [ + {"desc": "frontal", "count": road_sums["given"]}, + {"desc": "others", "count": road_sums["others"]}, + ], self.ALL_ROADS_SUBTITLE: [ {"desc": "frontal", "count": all_roads_sums["given"]}, - {"desc": "others", "count": all_roads_sums["others"]} - ] + {"desc": "others", "count": all_roads_sums["others"]}, + ], } return res @@ -437,7 +439,8 @@ def is_included(self) -> bool: else: raise ValueError all_total = all_h2h + all_others - return (segment_h2h > 0 and (segment_h2h/segment_total) > all_h2h/all_total) + return segment_h2h > 0 and (segment_h2h / segment_total) > all_h2h / all_total + # adding calls to _() for pybabel extraction _("others") @@ -450,7 +453,6 @@ class AccidentCountByAccidentTypeWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 6 def generate_items(self) -> None: self.items = AccidentCountByAccidentTypeWidget.get_accident_count_by_accident_type( @@ -484,7 +486,6 @@ class AccidentsHeatMapWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 7 def generate_items(self) -> None: accidents_heat_map_filters = self.request_params.location_info.copy() @@ -515,20 +516,20 @@ def localize_items(request_params: RequestParams, items: Dict) -> Dict: items["data"]["text"] = { "title": _("Fatal and severe accidents heat map") + " " - + segment_dictionary[request_params.location_info["road_segment_name"]] + + segment_dictionary[request_params.location_info.get("road_segment_name", "")] } return items + @register class AccidentCountByAccidentYearWidget(Widget): name: str = "accident_count_by_accident_year" def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 8 self.text = { "title": "כמות התאונות לפי שנה במקטע " - + self.request_params.location_info["road_segment_name"] + + self.request_params.location_info.get("road_segment_name", "") } def generate_items(self) -> None: @@ -548,10 +549,9 @@ class InjuredCountByAccidentYearWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 9 self.text = { "title": "נפגעים בתאונות במקטע " - + self.request_params.location_info["road_segment_name"] + + self.request_params.location_info.get("road_segment_name", "") } def generate_items(self) -> None: @@ -571,7 +571,6 @@ class AccidentCountByDayNightWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 10 self.text = {"title": "כמות תאונות ביום ובלילה"} def generate_items(self) -> None: @@ -591,7 +590,6 @@ class AccidentCountByHourWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 11 self.text = {"title": "כמות תאונות לפי שעה"} def generate_items(self) -> None: @@ -611,7 +609,6 @@ class AccidentCountByRoadLightWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 12 self.text = {"title": "כמות תאונות לפי תאורה"} def generate_items(self) -> None: @@ -630,7 +627,6 @@ class TopRoadSegmentsAccidentsPerKmWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 13 self.text = { "title": "תאונות לכל ק״מ כביש על פי מקטע בכביש " + str(int(self.request_params.location_info["road1"])) @@ -683,7 +679,6 @@ class InjuredCountPerAgeGroupWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 14 def generate_items(self) -> None: self.items = InjuredCountPerAgeGroupWidget.filter_and_group_injured_count_per_age_group( @@ -762,7 +757,7 @@ def defaultdict_int_factory() -> Callable: def localize_items(request_params: RequestParams, items: Dict) -> Dict: items["data"]["text"] = { "title": _("Injury severity per age group in ") - + request_params.location_info["road_segment_name"] + + request_params.location_info.get("road_segment_name", "") } return items @@ -773,11 +768,11 @@ class VisionZeroWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 15 def generate_items(self) -> None: self.items = ["vision_zero_2_plus_1"] + @register class Road2Plus1Widget(Widget): name: str = "vision_zero_2_plus_1" @@ -789,13 +784,17 @@ def __init__(self, request_params: RequestParams): def generate_items(self) -> None: self.items = ["vision_zero_2_plus_1"] + @register class AccidentCountByDriverTypeWidget(Widget): name: str = "accident_count_by_driver_type" def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 16 + self.text = { + "title": "מעורבות נהגים בתאונות לפי סוג במקטע " + + self.request_params.location_info.get("road_segment_name", "") + } def generate_items(self) -> None: self.items = AccidentCountByDriverTypeWidget.count_accidents_by_driver_type( @@ -845,7 +844,7 @@ def localize_items(request_params: RequestParams, items: Dict) -> Dict: ) items["data"]["text"] = { "title": _("accident count by driver type ") - + request_params.location_info["road_segment_name"] + + request_params.location_info.get("road_segment_name", "") } return items @@ -856,7 +855,6 @@ class AccidentCountByCarTypeWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 17 def generate_items(self) -> None: self.items = ( @@ -952,13 +950,16 @@ def localize_items(request_params: RequestParams, items: Dict) -> Dict: try: item["car_type"] = _(VehicleCategory(item["car_type"]).get_english_display_name()) except ValueError: - logging.exception(f'AccidentCountByCarType.localize_items: item:{item}') - base_title = _("comparing vehicle type percentage in accidents in" - " {} " - "relative to national average") + logging.exception(f"AccidentCountByCarType.localize_items: item:{item}") + base_title = _( + "comparing vehicle type percentage in accidents in" + " {} " + "relative to national average" + ) items["data"]["text"] = { "title": base_title.format( - segment_dictionary[request_params.location_info["road_segment_name"]]) + segment_dictionary[request_params.location_info.get("road_segment_name", "")] + ) } return items @@ -969,7 +970,6 @@ class InjuredAccidentsWithPedestriansWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 18 self.text = {"title": "נפגעים הולכי רגל ברחוב ז׳בוטינסקי, פתח תקווה"} # noinspection PyMethodMayBeStatic @@ -1056,7 +1056,6 @@ class AccidentSeverityByCrossLocationWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 19 self.text = {"title": "הולכי רגל הרוגים ופצועים קשה ברחוב בן יהודה, תל אביב"} # noinspection PyMethodMayBeStatic @@ -1097,7 +1096,6 @@ class MotorcycleAccidentsVsAllAccidentsWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 20 self.road_number: str = request_params.location_info["road1"] self.text = { "title": f"תאונות אופנועים קשות וקטלניות בכביש {int(self.road_number)} בהשוואה לכל הארץ" @@ -1109,8 +1107,9 @@ def generate_items(self) -> None: ) @staticmethod - def motorcycle_accidents_vs_all_accidents(start_time: datetime.date, - end_time: datetime.date, road_number: str) -> List: + def motorcycle_accidents_vs_all_accidents( + start_time: datetime.date, end_time: datetime.date, road_number: str + ) -> List: location_label = "location" location_other = "שאר הארץ" location_road = f"כביש {int(road_number)}" @@ -1131,7 +1130,9 @@ def motorcycle_accidents_vs_all_accidents(start_time: datetime.date, case_vehicle = case( [ ( - InvolvedMarkerView.involve_vehicle_type.in_(VehicleCategory.MOTORCYCLE.get_codes()), + InvolvedMarkerView.involve_vehicle_type.in_( + VehicleCategory.MOTORCYCLE.get_codes() + ), literal_column(f"'{vehicle_motorcycle}'"), ) ], @@ -1214,7 +1215,6 @@ class AccidentCountPedestriansPerVehicleStreetVsAllWidget(Widget): def __init__(self, request_params: RequestParams): Widget.__init__(self, request_params, type(self).name) - self.rank = 21 self.text = { "title": _( "Pedestrian Injuries on Ben Yehuda Street in Tel Aviv by Type of hitting Vehicle, Compared to Urban Accidents Across the country" @@ -1251,7 +1251,6 @@ class TopRoadSegmentsAccidentsWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 22 self.text = {"title": "5 המקטעים עם כמות התאונות הגדולה ביותר"} def generate_items(self) -> None: @@ -1274,7 +1273,6 @@ class PedestrianInjuredInJunctionsWidget(Widget): def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) - self.rank = 23 self.text = {"title": "מספר נפגעים הולכי רגל בצמתים - רחוב בן יהודה, תל אביב"} def generate_items(self) -> None: @@ -1297,8 +1295,9 @@ def __init__(self, request_params: RequestParams): super().__init__(request_params, type(self).name) self.road_number: str = request_params.location_info["road1"] # WIP: change rank, text by vehicle type - self.rank = 24 - self.text = {"title": f"סוגי תאונות אופנועים בכביש {int(self.road_number)} בהשוואה לכל הארץ"} + self.text = { + "title": f"סוגי תאונות אופנועים בכביש {int(self.road_number)} בהשוואה לכל הארץ" + } def generate_items(self) -> None: self.items = AccidentTypeVehicleTypeRoadComparisonWidget.accident_type_road_vs_all_count( @@ -1306,25 +1305,32 @@ def generate_items(self) -> None: ) @staticmethod - def accident_type_road_vs_all_count(start_time: datetime.date, - end_time: datetime.date, road_number: str) -> List: + def accident_type_road_vs_all_count( + start_time: datetime.date, end_time: datetime.date, road_number: str + ) -> List: num_accidents_label = "num_of_accidents" location_all = "כל הארץ" location_road = f"כביש {int(road_number)}" vehicle_types = VehicleCategory.MOTORCYCLE.get_codes() # WIP: change by vehicle type - all_roads_query = AccidentTypeVehicleTypeRoadComparisonWidget.get_accident_count_by_vehicle_type_query( - start_time, end_time, num_accidents_label, vehicle_types) + all_roads_query = ( + AccidentTypeVehicleTypeRoadComparisonWidget.get_accident_count_by_vehicle_type_query( + start_time, end_time, num_accidents_label, vehicle_types + ) + ) all_roads_query_result = run_query(all_roads_query) all_roads_sum_accidents = 0 all_roads_map = {} for record in all_roads_query_result: all_roads_sum_accidents += record[num_accidents_label] - all_roads_map[record[VehicleMarkerView.accident_type.name]] = record[num_accidents_label] + all_roads_map[record[VehicleMarkerView.accident_type.name]] = record[ + num_accidents_label + ] - road_query = all_roads_query.filter((VehicleMarkerView.road1 == road_number) | - (VehicleMarkerView.road2 == road_number)) + road_query = all_roads_query.filter( + (VehicleMarkerView.road1 == road_number) | (VehicleMarkerView.road2 == road_number) + ) road_query_result = run_query(road_query) road_sum_accidents = 0 types_to_report = [] @@ -1332,32 +1338,50 @@ def accident_type_road_vs_all_count(start_time: datetime.date, road_sum_accidents += record[num_accidents_label] for record in road_query_result: - if len(types_to_report) == AccidentTypeVehicleTypeRoadComparisonWidget.MAX_ACCIDENT_TYPES_TO_RETURN: + if ( + len(types_to_report) + == AccidentTypeVehicleTypeRoadComparisonWidget.MAX_ACCIDENT_TYPES_TO_RETURN + ): break accident_type = record[VehicleMarkerView.accident_type.name] - types_to_report.append({VehicleMarkerView.accident_type.name: accident_type, - location_road: record[num_accidents_label] / road_sum_accidents, - location_all: all_roads_map[accident_type] / all_roads_sum_accidents}) + types_to_report.append( + { + VehicleMarkerView.accident_type.name: accident_type, + location_road: record[num_accidents_label] / road_sum_accidents, + location_all: all_roads_map[accident_type] / all_roads_sum_accidents, + } + ) return types_to_report @staticmethod - def get_accident_count_by_vehicle_type_query(start_time: datetime.date, - end_time: datetime.date, - num_accidents_label: str, vehicle_types: List[int])\ - -> db.session.query: - return (get_query(table_obj=VehicleMarkerView, start_time=start_time, end_time=end_time, - filters={VehicleMarkerView.vehicle_type.name: vehicle_types}) - .with_entities(VehicleMarkerView.accident_type, - func.count(distinct(VehicleMarkerView.provider_and_id)).label(num_accidents_label)) - .group_by(VehicleMarkerView.accident_type) - .order_by(desc(num_accidents_label)) - ) + def get_accident_count_by_vehicle_type_query( + start_time: datetime.date, + end_time: datetime.date, + num_accidents_label: str, + vehicle_types: List[int], + ) -> db.session.query: + return ( + get_query( + table_obj=VehicleMarkerView, + start_time=start_time, + end_time=end_time, + filters={VehicleMarkerView.vehicle_type.name: vehicle_types}, + ) + .with_entities( + VehicleMarkerView.accident_type, + func.count(distinct(VehicleMarkerView.provider_and_id)).label(num_accidents_label), + ) + .group_by(VehicleMarkerView.accident_type) + .order_by(desc(num_accidents_label)) + ) @staticmethod def localize_items(request_params: RequestParams, items: Dict) -> Dict: for item in items["data"]["items"]: try: - item[VehicleMarkerView.accident_type.name] = _(english_accident_type_dict[item["accident_type"]]) + item[VehicleMarkerView.accident_type.name] = _( + english_accident_type_dict[item["accident_type"]] + ) except KeyError: logging.exception( f"AccidentTypeVehicleTypeRoadComparisonWidget.localize_items: Exception while translating {item}." @@ -1367,8 +1391,7 @@ def localize_items(request_params: RequestParams, items: Dict) -> Dict: def run_query(query: db.session.query) -> Dict: # pylint: disable=no-member - return pd.read_sql_query(query.statement, query.session.bind) \ - .to_dict(orient="records") + return pd.read_sql_query(query.statement, query.session.bind).to_dict(orient="records") def extract_news_flash_location(news_flash_obj): @@ -1444,7 +1467,10 @@ def get_most_severe_accidents_with_entities( BE_CONST.CBS_ACCIDENT_TYPE_1_CODE, BE_CONST.CBS_ACCIDENT_TYPE_3_CODE, ] - filters["accident_severity"] = [BE_CONST.AccidentSeverity.FATAL, BE_CONST.AccidentSeverity.SEVERE] + filters["accident_severity"] = [ + BE_CONST.AccidentSeverity.FATAL, + BE_CONST.AccidentSeverity.SEVERE, + ] query = get_query(table_obj, filters, start_time, end_time) query = query.with_entities(*entities) query = query.order_by(getattr(table_obj, "accident_timestamp").desc()) @@ -1458,7 +1484,7 @@ def get_most_severe_accidents_table_title(location_info): return ( _("Most severe accidents in segment") + " " - + segment_dictionary[location_info["road_segment_name"]] + + segment_dictionary[location_info.get("road_segment_name", "")] ) @@ -1491,7 +1517,7 @@ def get_news_flash_location_text(news_flash_obj: NewsFlash): road1 = str(int(nf["road1"])) if nf["road1"] else "" road2 = str(int(nf["road2"])) if nf["road2"] else "" street1_hebrew = nf["street1_hebrew"] if nf["street1_hebrew"] else "" - road_segment_name = nf["road_segment_name"] if nf["road_segment_name"] else "" + road_segment_name = nf["road_segment_name"] if nf.get("road_segment_name", "") else "" if resolution == "כביש בינעירוני" and road1 and road_segment_name: res = "כביש " + road1 + " במקטע " + road_segment_name elif resolution == "עיר" and not yishuv_name: @@ -1524,6 +1550,20 @@ def extract_news_flash_obj(news_flash_id): return news_flash_obj +def get_newsflash_features(newsflash_id: int) -> Optional[NewsflashFeatures]: + by_newsflash_id_and_current_version = [ + NewsflashFeatures.newsflash_id == newsflash_id, + NewsflashFeatures.version == NewsflashFeatureGenerator.VERSION, + ] + latest = NewsflashFeatures.timestamp.desc() + + return ( + db.session.query(NewsflashFeatures) + .filter(*by_newsflash_id_and_current_version) + .order_by(latest) + ).first() + + def sum_road_accidents_by_specific_type(road_data, field_name): dict_merge = defaultdict(int) dict_merge[field_name] = 0 @@ -1540,7 +1580,7 @@ def sum_road_accidents_by_specific_type(road_data, field_name): def convert_roads_fatal_accidents_to_frontend_view(data_dict): data_list = [] for key, value in data_dict.items(): - if key == BE_CONST.AccidentType.HEAD_ON_FRONTAL_COLLISION: + if key == head_on_collisions_comparison_dict["head_to_head_collision"]: data_list.append( {"desc": head_on_collisions_comparison_dict["head_to_head"], "count": value} ) @@ -1563,20 +1603,22 @@ def get_latest_accident_date(table_obj, filters): def generate_widgets(request_params: RequestParams, to_cache: bool = True) -> List[Widget]: - widgets = [] - # noinspection PyArgumentList - for w in get_widget_factories(): - widget: Widget = w(request_params) + # for w in WidgetId: + widgets_by_score_and_order = [] # (rank, tie-breaker, widget) will be sorted by rank + sort_sentinel = 0 # Used as tie-breaker in sorting, to avoid comparing widgets + for widget_cls in widgets_dict.values(): + # widget: Optional[Widget] = create_widget(w, request_params) + # TODO this may break - we can't guarantee init signature. Better to use a factory here + widget: Widget = widget_cls(request_params) + if widget.is_in_cache() == to_cache: - widgets.append(widget) - logging.debug(f"name:{widget.name}, class:{get_widget_class_by_name(widget.name)}") - for w in widgets: - w.generate_items() - filtered_widgets = [] - for w in widgets: - if w.is_included(): - filtered_widgets.append(w) - return filtered_widgets + rank = widget.calc_rank(request_params) + # Adding the sentinel for strong ordering, because when sorting the tuples, we can't + # sort by the third tuple member - widget + widgets_by_score_and_order.append((rank, sort_sentinel, widget)) + sort_sentinel += 1 + widgets_by_score_and_order.sort() + return [w[2] for w in widgets_by_score_and_order] # Extracting only widgets, in order def get_request_params( @@ -1591,6 +1633,15 @@ def get_request_params( news_flash_obj: Optional[NewsFlash] = extract_news_flash_obj(news_flash_id) if news_flash_obj is None: return None + assert news_flash_obj # mypy + + # Features are generated on-demand. If no features are found for this newsflash, generate and store + features = get_newsflash_features(newsflash_id=news_flash_obj.id) + if features is None: + features = NewsflashFeatureGenerator.generate(news_flash_obj) + db.session.add(features) + db.session.commit() + location_info = extract_news_flash_location(news_flash_obj) if location_info is None: return None @@ -1614,6 +1665,7 @@ def get_request_params( request_params = RequestParams( news_flash_obj=news_flash_obj, + newsflash_features=features, years_ago=number_of_years_ago, location_text=location_text, location_info=location_info, diff --git a/anyway/models.py b/anyway/models.py index c8d58bcaa..a603fe34e 100755 --- a/anyway/models.py +++ b/anyway/models.py @@ -819,7 +819,7 @@ def get_id(self): class NewsFlash(Base): __tablename__ = "news_flash" - id = Column(BigInteger(), primary_key=True) + id = Column(BigInteger(), primary_key=True, unique=True) accident = Column(Boolean(), nullable=False) author = Column(Text(), nullable=True) date = Column(TIMESTAMP(), nullable=True) @@ -885,6 +885,46 @@ def get_id(self): return self.id +class NewsflashFeatures(Base): + __tablename__ = "newsflash_features" + + id = Column(BigInteger(), primary_key=True) + newsflash_id = Column(BigInteger(), ForeignKey(NewsFlash.id)) + + """ + version is the feature calculation version number. Can be used to compare results between versions, and + also to re-calculate results + """ + version = Column(Integer(), nullable=False) + + """When did this calculation take place""" + timestamp = Column(DateTime(), nullable=False) + + is_urban = Column(Boolean()) + + def serialize(self): + return { + "id": self.id, + "newsflash_id": self.newsflash_id, + "timestamp": self.timestamp, + "version": self.version, + "is_urban": self.is_urban, + } + + # Flask-Login integration + def is_authenticated(self): + return True + + def is_active(self): + return True + + def is_anonymous(self): + return False + + def get_id(self): + return self.id + + class City(Base): __tablename__ = "cities" id = Column(Integer(), primary_key=True) diff --git a/anyway/parsers/cbs/executor.py b/anyway/parsers/cbs/executor.py index be09938e0..283aeace8 100644 --- a/anyway/parsers/cbs/executor.py +++ b/anyway/parsers/cbs/executor.py @@ -1000,7 +1000,9 @@ def create_tables(): conn.execute("INSERT INTO involved_hebrew " + VIEWS.INVOLVED_HEBREW_VIEW) conn.execute("INSERT INTO vehicles_hebrew " + VIEWS.VEHICLES_HEBREW_VIEW) conn.execute("INSERT INTO vehicles_markers_hebrew " + VIEWS.VEHICLES_MARKERS_HEBREW_VIEW) - conn.execute("INSERT INTO involved_markers_hebrew " + VIEWS.INVOLVED_HEBREW_MARKERS_HEBREW_VIEW) + conn.execute( + "INSERT INTO involved_markers_hebrew " + VIEWS.INVOLVED_HEBREW_MARKERS_HEBREW_VIEW + ) logging.info("Created DB Hebrew Tables") diff --git a/anyway/parsers/infographics_data_cache_updater.py b/anyway/parsers/infographics_data_cache_updater.py index 92ed24f5e..93611150b 100755 --- a/anyway/parsers/infographics_data_cache_updater.py +++ b/anyway/parsers/infographics_data_cache_updater.py @@ -28,7 +28,7 @@ def is_in_cache(nf): ) -def add_news_flash_to_cache(news_flash): +def add_news_flash_to_cache(news_flash: NewsFlash): try: if not is_cache_eligible(news_flash): logging.debug( diff --git a/anyway/parsers/news_flash_db_adapter.py b/anyway/parsers/news_flash_db_adapter.py index 4f1346a4f..88e002f1a 100644 --- a/anyway/parsers/news_flash_db_adapter.py +++ b/anyway/parsers/news_flash_db_adapter.py @@ -1,5 +1,6 @@ import datetime import logging + import pandas as pd from flask_sqlalchemy import SQLAlchemy from anyway.parsers import infographics_data_cache_updater @@ -8,7 +9,6 @@ # fmt: off - def init_db() -> "DBAdapter": from anyway.app_and_db import db return DBAdapter(db) @@ -64,8 +64,9 @@ def remove_duplicate_rows(self): def insert_new_newsflash(self, newsflash: NewsFlash) -> None: logging.info("Adding newsflash, is accident: {}, date: {}" .format(newsflash.accident, newsflash.date)) - self.db.session.add(newsflash) - self.db.session.commit() + with self.db.session as s: + s.add(newsflash) + s.commit() infographics_data_cache_updater.add_news_flash_to_cache(newsflash) def get_newsflash_by_id(self, id): diff --git a/anyway/rules/__init__.py b/anyway/rules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/anyway/rules/newsflash_feature_generator.py b/anyway/rules/newsflash_feature_generator.py new file mode 100644 index 000000000..a15919eac --- /dev/null +++ b/anyway/rules/newsflash_feature_generator.py @@ -0,0 +1,23 @@ +import datetime + +from anyway.models import NewsFlash, NewsflashFeatures + + +class NewsflashFeatureGenerator: + + VERSION = 1 + + def __init__(self): + pass + + @classmethod + def generate(cls, newsflash: NewsFlash) -> NewsflashFeatures: + result = NewsflashFeatures() + result.newsflash_id = newsflash.id + result.version = NewsflashFeatureGenerator.VERSION + result.timestamp = datetime.datetime.utcnow() + + # TODO replace this with a real implementation + result.is_urban = newsflash.id % 2 == 0 + + return result diff --git a/anyway/utilities.py b/anyway/utilities.py index db6750aad..7e34805b4 100644 --- a/anyway/utilities.py +++ b/anyway/utilities.py @@ -204,9 +204,27 @@ def chunks(l, n): def parse_age_from_range(age_range: int) -> typing.Optional[typing.Tuple[int, int]]: # Convert from 'age_group' field in the table 'involved_markers_hebrew' to age range numbers - ret_age_code_to_age_range = {1: (0, 4), 2: (5, 9), 3: (10, 14), 4: (15, 19), 5: (20, 24), 6: (25, 29), 7: (30, 34), - 8: (35, 39), 9: (40, 44), 10: (45, 49), 11: (50, 54), 12: (55,59), 13: (60, 64), - 14: (65, 69),15: (70, 74), 16: (75, 79), 17: (80, 84), 18: (85, 200), 99: None} + ret_age_code_to_age_range = { + 1: (0, 4), + 2: (5, 9), + 3: (10, 14), + 4: (15, 19), + 5: (20, 24), + 6: (25, 29), + 7: (30, 34), + 8: (35, 39), + 9: (40, 44), + 10: (45, 49), + 11: (50, 54), + 12: (55, 59), + 13: (60, 64), + 14: (65, 69), + 15: (70, 74), + 16: (75, 79), + 17: (80, 84), + 18: (85, 200), + 99: None, + } return ret_age_code_to_age_range[age_range] diff --git a/anyway/vehicle_type.py b/anyway/vehicle_type.py index 7917bf44a..b3b1f8bc5 100644 --- a/anyway/vehicle_type.py +++ b/anyway/vehicle_type.py @@ -82,8 +82,10 @@ def to_type_code(db_val: Union[float, int]) -> int: elif isinstance(db_val, int): return db_val else: - logging.error(f"VehicleType.fo_type_code: unknown value: {db_val}({type(db_val)})" - '. returning OTHER_AND_UNKNOWN') + logging.error( + f"VehicleType.fo_type_code: unknown value: {db_val}({type(db_val)})" + ". returning OTHER_AND_UNKNOWN" + ) return VehicleType.OTHER_AND_UNKNOWN.value @@ -128,7 +130,7 @@ def get_codes(self) -> List[int]: VehicleCategory.LIGHT_ELECTRIC: [ VehicleType.ELECTRIC_SCOOTER, VehicleType.MOBILITY_SCOOTER, - VehicleType.ELECTRIC_BIKE + VehicleType.ELECTRIC_BIKE, ], VehicleCategory.CAR: [VehicleType.CAR, VehicleType.TAXI], VehicleCategory.LARGE: [ @@ -151,8 +153,16 @@ def get_codes(self) -> List[int]: VehicleType.MOTORCYCLE_250_TO_500, VehicleType.MOTORCYCLE_ABOVE_500, ], - VehicleCategory.BICYCLE_AND_SMALL_MOTOR: [VehicleType.BIKE, VehicleType.ELECTRIC_SCOOTER, VehicleType.ELECTRIC_BIKE], - VehicleCategory.OTHER: [VehicleType.BIKE, VehicleType.TRAIN, VehicleType.OTHER_AND_UNKNOWN] + VehicleCategory.BICYCLE_AND_SMALL_MOTOR: [ + VehicleType.BIKE, + VehicleType.ELECTRIC_SCOOTER, + VehicleType.ELECTRIC_BIKE, + ], + VehicleCategory.OTHER: [ + VehicleType.BIKE, + VehicleType.TRAIN, + VehicleType.OTHER_AND_UNKNOWN, + ], } return list(map(lambda x: x.value, category_vehicle_types[self])) diff --git a/anyway/views/news_flash/api.py b/anyway/views/news_flash/api.py index 803eb7306..eb40217e3 100644 --- a/anyway/views/news_flash/api.py +++ b/anyway/views/news_flash/api.py @@ -13,6 +13,7 @@ DEFAULT_OFFSET_REQ_PARAMETER = 0 DEFAULT_LIMIT_REQ_PARAMETER = 100 + @user_optional def news_flash(): logging.debug("getting news flash") diff --git a/tests/test_infographic_api.py b/tests/test_infographic_api.py index aea0dc40b..ebd68d595 100644 --- a/tests/test_infographic_api.py +++ b/tests/test_infographic_api.py @@ -56,8 +56,11 @@ def get_infographic_data(): def delete_new_infographic_data(new_infographic_data_id): - sql_delete = f"DELETE FROM news_flash where id = {new_infographic_data_id}" - db.session.execute(sql_delete) + delete_newsflash = f"DELETE FROM news_flash where id = {new_infographic_data_id}" + # Deleting linked row by foreign key as well + delete_features = f"DELETE FROM newsflash_features where newsflash_id = {new_infographic_data_id}" + db.session.execute(delete_features) + db.session.execute(delete_newsflash) db.session.commit()