Fixes post processing. Add more detailed stats for custom exports.

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.5.18 (2025-03-04)
+
+### Fix
+
+- **post_processing**: Fixes wrong stats and add more detailed HTML stats for custom exports. Disable transliterations for now
+
 ## 1.5.17 (2025-02-18)
 
 ### Fix

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "raw-data-api"
-version = "1.5.17"
+version = "1.5.18"
 description = "Set of high-performant APIs for transforming and exporting OpenStreetMap (OSM) data in different GIS file formats."
 readme = "README.md"
 authors = [

requirements.txt

@@ -27,7 +27,7 @@ humanize==4.9.0
 python-slugify==8.0.1
 geomet==1.1.0
 PyYAML==6.0.1
-geojson-stats==0.2.6
+geojson-stats==0.2.7
 transliterate==1.10.2
 
 ## documentation

src/app.py

@@ -1526,8 +1526,7 @@ def process_export_format(export_format):
                         "include_stats_html": self.params.include_stats_html,
                     }
                 )
-                post_processor.init()
-                post_processor.custom(
+                post_processor.stats(
                     category_name=category_name,
                     export_format_path=export_format_path,
                     export_filename=export_filename,

src/post_processing/processor.py

@@ -1,6 +1,5 @@
-import json
-from .transliterator import Transliterator
-from .geojson_stats import GeoJSONStats
+from geojson_stats.stats import Stats, Config
+from geojson_stats.html import Html
 import os
 import pathlib
 
@@ -16,84 +15,56 @@
 class PostProcessor:
     """Used for post-process GeoJSON files"""
 
-    options = {}
-    filters = {}
-    functions = []
-
     def __init__(self, options, *args, **kwargs):
         self.options = options
 
-    def post_process_line(self, line: str):
-        """
-        Parses line, run functions over it and returns it
-        """
-
-        line_object = json.loads(line)
-
-        for fn in self.functions:
-            fn(line_object)
-
-        return json.dumps(line_object)
-
     def get_categories_config(self, category_name):
         """
         Get configuration for categories
         """
         config = CATEGORIES_CONFIG.get(category_name)
         return config if config else CATEGORIES_CONFIG["default"]
 
-    def custom(
+    def stats(
         self, category_name, export_format_path, export_filename, file_export_path
     ):
         """
         Post-process custom exports
         """
-        self.geoJSONStats.config.properties_prop = "properties"
 
+        # Get stats config
         category_config = self.get_categories_config(category_name)
-        category_tag = category_config["tag"]
-        self.geoJSONStats.config.length = category_config["length"]
-        self.geoJSONStats.config.area = category_config["area"]
+        config = Config(
+            clean=True,
+            length=category_config["length"],
+            area=category_config["area"],
+            keys=category_config["tag"],
+            value_keys=category_config["tag"],
+        )
 
         if self.options["include_stats"]:
+            # Generate stats
             path_input = os.path.join(export_format_path, f"{export_filename}.geojson")
-            path_output = os.path.join(
-                export_format_path, f"{export_filename}-post.geojson"
-            )
+            stats = Stats(config)
+            stats.process_file_stream(path_input)
 
-            with open(path_input, "r") as input_file, open(
-                path_output, "w"
-            ) as output_file:
-                for line in input_file:
-                    comma = False
-                    if line.startswith('{ "type": "Feature"'):
-                        json_string = ""
-                        if line[-2:-1] == ",":
-                            json_string = line[:-2]
-                            comma = True
-                        else:
-                            json_string = line
-                        line = self.post_process_line(json_string)
-                    if self.options["include_translit"]:
-                        if comma:
-                            output_file.write(line + ",")
-                        else:
-                            output_file.write(line)
+            # Remove redundant stats
+            del stats.results.key["osm_id"]
+            del stats.results.key["osm_type"]
 
-            if self.options.get("include_translit"):
-                os.remove(path_input)
-                os.rename(path_output, path_input)
-            else:
-                os.remove(path_output)
+            stats_json = stats.json()
 
-            geojson_stats_json = json.dumps(self.geoJSONStats.dict())
+            # Save raw stats
             with open(
                 os.path.join(file_export_path, "stats.json"),
                 "w",
             ) as f:
-                f.write(geojson_stats_json)
+                f.write(stats_json)
 
+            # Save HTML stats
             if self.options.get("include_stats_html"):
+                # Get template
+                category_tag = category_config["tag"]
                 tpl = (
                     "stats_{category_tag}".format(category_tag=category_tag)
                     if category_tag
@@ -104,22 +75,13 @@ def custom(
                     project_root,
                     "{tpl}_tpl.html".format(tpl=tpl),
                 )
-                geojson_stats_html = self.geoJSONStats.html(
-                    tpl_path, {"title": f"{export_filename}.geojson"}
+
+                # Generate HTML
+                geojson_stats_html = Html(
+                    tpl_path, stats, {"title": f"{export_filename}.geojson"}
                 ).build()
+
+                # Save HTML file
                 upload_html_path = os.path.join(file_export_path, "stats-summary.html")
                 with open(upload_html_path, "w") as f:
                     f.write(geojson_stats_html)
-
-    def init(self):
-        """
-        Initialize post-processor
-        """
-
-        if self.options.get("include_stats"):
-            self.geoJSONStats = GeoJSONStats(self.filters)
-            self.functions.append(self.geoJSONStats.raw_data_line_stats)
-
-        if self.options.get("include_translit"):
-            self.transliterator = Transliterator()
-            self.functions.append(self.transliterator.translit)

src/post_processing/stats_building_tpl.html

@@ -3,31 +3,31 @@
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/hot.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
     <link href="https://fonts.googleapis.com/css2?family=Archivo:ital,wght@0,100..900;1,100..900&display=swap" rel="stylesheet">
     <title>HOT Export Stats</title>
     <style type="text/css">
+
         :root,
         :host,
         .hot-theme-light {
+            --hot-color-red-700: #C53639;
+            --hot-color-gray-950: #2C3038;
             --hot-font-sans: Archivo, -apple-system, Roboto, Helvetica, Arial, sans-serif,
   'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';
             --hot-font-size-2x-large: 2.25rem; /* 36px */
             --hot-font-size-medium: 1rem; /* 16px */
             --hot-font-size-small: 0.875rem; /* 14px */
-            --hot-font-weight-normal: 400;
             --hot-border-radius-medium: 0.25rem; /* 4px */
             --hot-color-neutral-0: #fff;
-            --hot-color-red-700: #C53639;
-            --hot-color-gray-50: #F3F3F3;
             --hot-color-gray-200: #C4C3C5;
-            --hot-color-gray-100: #E1E0E1;
-            --hot-color-gray-400: #9A969B;
-            --hot-color-gray-950: #2C3038;
+            --hot-color-gray-50: #F3F3F3;
             --hot-spacing-medium: 1rem;
             --hot-spacing-x-small: 0.5rem; /* 8px */
+            --hot-color-gray-100: #E1E0E1;
         }
         body {
             font-family: var(--hot-font-sans);
@@ -42,11 +42,6 @@
             vertical-align: baseline;
             background: transparent;
         }
-        h2 {
-            margin-left: var(--hot-spacing-medium);
-            color: var(--hot-color-gray-400);
-            font-weight: var(--hot-font-weight-normal);
-        }
         .container {
             display: flex;
             width: 100%;
@@ -101,10 +96,9 @@
   </head>
   <body>
     <div id="root">
-        <h2>${title}</h2>
         <div class="container">
             <div class="box featured">
-                <h3>${area}</h3>
+                <h3>${key_building_area}</h3>
                 <h4>Km2 of buildings</h4>
             </div>
             <div class="box">
@@ -115,6 +109,9 @@ <h4>Number of Features</h4>
             <div class="box">
                 <h3>${languages_count}</h3>
                 <h4>Languages Available</h4>
+                <p>
+                    ${languages_list}
+                </p>
                 <p>Including local language and english</p>
             </div>
         </div>
@@ -128,11 +125,6 @@ <h4>Languages Available</h4>
                     </tr>
                 </thead>
                 <tbody>
-                    <tr>
-                        <td>Total features</td>
-                        <td>${count}</td>
-                        <td>100%</td>
-                    </tr>
                     <tr>
                         <td>${key_0}</td>
                         <td>${key_0_count}</td>
@@ -165,6 +157,40 @@ <h4>Languages Available</h4>
                     </tr>
                 </tbody>
             </table>
+            <table class="table box">
+                <thead>
+                    <tr>
+                        <th>Building</th>
+                        <th>Count</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    <tr>
+                        <td>${key_building_value_0}</td>
+                        <td>${key_building_value_0_count}</td>
+                    </tr>
+                    <tr>
+                        <td>${key_building_value_1}</td>
+                        <td>${key_building_value_1_count}</td>
+                    </tr>
+                    <tr>
+                        <td>${key_building_value_2}</td>
+                        <td>${key_building_value_2_count}</td>
+                    </tr>
+                    <tr>
+                        <td>${key_building_value_3}</td>
+                        <td>${key_building_value_3_count}</td>
+                    </tr>
+                    <tr>
+                        <td>${key_building_value_4}</td>
+                        <td>${key_building_value_4_count}</td>
+                    </tr>
+                    <tr>
+                        <td>${key_building_value_5}</td>
+                        <td>${key_building_value_5_count}</td>
+                    </tr>
+                </tbody>
+            </table>
         </div>
     </div>
   </body>

src/post_processing/stats_highway_tpl.html

@@ -114,6 +114,9 @@ <h4>Number of Features</h4>
             <div class="box">
                 <h3>${languages_count}</h3>
                 <h4>Languages Available</h4>
+                <p>
+                    ${languages_list}
+                </p>
                 <p>Including local language and english</p>
             </div>
         </div>
@@ -127,11 +130,6 @@ <h4>Languages Available</h4>
                     </tr>
                 </thead>
                 <tbody>
-                    <tr>
-                        <td>Total features</td>
-                        <td>${count}</td>
-                        <td>100%</td>
-                    </tr>
                     <tr>
                         <td>${key_0}</td>
                         <td>${key_0_count}</td>
@@ -164,6 +162,40 @@ <h4>Languages Available</h4>
                     </tr>
                 </tbody>
             </table>
+            <table class="table box">
+                <thead>
+                    <tr>
+                        <th>Highway</th>
+                        <th>Count</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    <tr>
+                        <td>${key_highway_value_0}</td>
+                        <td>${key_highway_value_0_count}</td>
+                    </tr>
+                    <tr>
+                        <td>${key_highway_value_1}</td>
+                        <td>${key_highway_value_1_count}</td>
+                    </tr>
+                    <tr>
+                        <td>${key_highway_value_2}</td>
+                        <td>${key_highway_value_2_count}</td>
+                    </tr>
+                    <tr>
+                        <td>${key_highway_value_3}</td>
+                        <td>${key_highway_value_3_count}</td>
+                    </tr>
+                    <tr>
+                        <td>${key_highway_value_4}</td>
+                        <td>${key_highway_value_4_count}</td>
+                    </tr>
+                    <tr>
+                        <td>${key_highway_value_5}</td>
+                        <td>${key_highway_value_5_count}</td>
+                    </tr>
+                </tbody>
+            </table>
         </div>
     </div>
   </body>