From 44c5dc278b19c635b5ef45568884d72606ff9b4e Mon Sep 17 00:00:00 2001 From: Emilio Mariscal Date: Tue, 4 Mar 2025 17:24:35 -0300 Subject: [PATCH 1/4] fix(post_processing): Fixes wrong stats and add more detailed HTML stats for custom exports. Disable transliterations for now --- src/app.py | 3 +- src/post_processing/geojson_stats.py | 24 ------ src/post_processing/processor.py | 96 ++++++--------------- src/post_processing/stats_building_tpl.html | 62 +++++++++---- src/post_processing/stats_highway_tpl.html | 42 +++++++-- src/post_processing/stats_railway_tpl.html | 42 +++++++-- src/post_processing/stats_tpl.html | 8 +- src/post_processing/stats_waterway_tpl.html | 42 +++++++-- 8 files changed, 186 insertions(+), 133 deletions(-) delete mode 100644 src/post_processing/geojson_stats.py diff --git a/src/app.py b/src/app.py index 4288694b..2d25739a 100644 --- a/src/app.py +++ b/src/app.py @@ -1526,8 +1526,7 @@ def process_export_format(export_format): "include_stats_html": self.params.include_stats_html, } ) - post_processor.init() - post_processor.custom( + post_processor.stats( category_name=category_name, export_format_path=export_format_path, export_filename=export_filename, diff --git a/src/post_processing/geojson_stats.py b/src/post_processing/geojson_stats.py deleted file mode 100644 index 6a5e612e..00000000 --- a/src/post_processing/geojson_stats.py +++ /dev/null @@ -1,24 +0,0 @@ -from geojson_stats.stats import Stats -from geojson_stats.html import Html - - -class GeoJSONStats(Stats): - """Used for collecting stats while processing GeoJSON files""" - - def __init__(self, filters, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.config.clean = True - self.config.properties_prop = "properties.tags" - - def raw_data_line_stats(self, json_object: dict): - """ - Process a GeoJSON line (for getting stats) and return that line - """ - self.get_object_stats(json_object) - - def html(self, tpl, tpl_params): - """ - Returns stats Html object, generated from stats data using a template - """ - return Html(tpl, self, tpl_params) diff --git a/src/post_processing/processor.py b/src/post_processing/processor.py index ce07dc15..497e78d6 100644 --- a/src/post_processing/processor.py +++ b/src/post_processing/processor.py @@ -1,6 +1,5 @@ -import json -from .transliterator import Transliterator -from .geojson_stats import GeoJSONStats +from geojson_stats.stats import Stats, Config +from geojson_stats.html import Html import os import pathlib @@ -16,25 +15,9 @@ class PostProcessor: """Used for post-process GeoJSON files""" - options = {} - filters = {} - functions = [] - def __init__(self, options, *args, **kwargs): self.options = options - def post_process_line(self, line: str): - """ - Parses line, run functions over it and returns it - """ - - line_object = json.loads(line) - - for fn in self.functions: - fn(line_object) - - return json.dumps(line_object) - def get_categories_config(self, category_name): """ Get configuration for categories @@ -42,58 +25,42 @@ def get_categories_config(self, category_name): config = CATEGORIES_CONFIG.get(category_name) return config if config else CATEGORIES_CONFIG["default"] - def custom( + def stats( self, category_name, export_format_path, export_filename, file_export_path ): """ Post-process custom exports """ - self.geoJSONStats.config.properties_prop = "properties" + # Get stats config category_config = self.get_categories_config(category_name) - category_tag = category_config["tag"] - self.geoJSONStats.config.length = category_config["length"] - self.geoJSONStats.config.area = category_config["area"] + config = Config( + clean=True, + length=category_config["length"], + area=category_config["area"], + keys=category_config["tag"], + value_keys=category_config["tag"], + ) if self.options["include_stats"]: + # Generate stats path_input = os.path.join(export_format_path, f"{export_filename}.geojson") - path_output = os.path.join( - export_format_path, f"{export_filename}-post.geojson" - ) - - with open(path_input, "r") as input_file, open( - path_output, "w" - ) as output_file: - for line in input_file: - comma = False - if line.startswith('{ "type": "Feature"'): - json_string = "" - if line[-2:-1] == ",": - json_string = line[:-2] - comma = True - else: - json_string = line - line = self.post_process_line(json_string) - if self.options["include_translit"]: - if comma: - output_file.write(line + ",") - else: - output_file.write(line) + stats = Stats(config) + stats.process_file_stream(path_input) + del stats.results.key["osm_id"] + stats_json = stats.json() - if self.options.get("include_translit"): - os.remove(path_input) - os.rename(path_output, path_input) - else: - os.remove(path_output) - - geojson_stats_json = json.dumps(self.geoJSONStats.dict()) + # Save raw stats with open( os.path.join(file_export_path, "stats.json"), "w", ) as f: - f.write(geojson_stats_json) + f.write(stats_json) + # Save HTML stats if self.options.get("include_stats_html"): + # Get template + category_tag = category_config["tag"] tpl = ( "stats_{category_tag}".format(category_tag=category_tag) if category_tag @@ -104,22 +71,13 @@ def custom( project_root, "{tpl}_tpl.html".format(tpl=tpl), ) - geojson_stats_html = self.geoJSONStats.html( - tpl_path, {"title": f"{export_filename}.geojson"} + + # Generate HTML + geojson_stats_html = Html( + tpl_path, stats, {"title": f"{export_filename}.geojson"} ).build() + + # Save HTML file upload_html_path = os.path.join(file_export_path, "stats-summary.html") with open(upload_html_path, "w") as f: f.write(geojson_stats_html) - - def init(self): - """ - Initialize post-processor - """ - - if self.options.get("include_stats"): - self.geoJSONStats = GeoJSONStats(self.filters) - self.functions.append(self.geoJSONStats.raw_data_line_stats) - - if self.options.get("include_translit"): - self.transliterator = Transliterator() - self.functions.append(self.transliterator.translit) diff --git a/src/post_processing/stats_building_tpl.html b/src/post_processing/stats_building_tpl.html index d63415b9..39943c85 100644 --- a/src/post_processing/stats_building_tpl.html +++ b/src/post_processing/stats_building_tpl.html @@ -3,31 +3,31 @@ + HOT Export Stats