From 871d2d3fa9e5dbc111eefde2719758b5ee09117c Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Wed, 1 Jul 2026 16:12:23 -0500 Subject: [PATCH 01/23] Add TippecanoeService and install tippecanoe in worker image --- devops/Dockerfile.worker | 12 +++ geoapi/services/tippecanoe.py | 78 +++++++++++++++++++ .../tests/services/test_tippecanoe_service.py | 76 ++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 geoapi/services/tippecanoe.py create mode 100644 geoapi/tests/services/test_tippecanoe_service.py diff --git a/devops/Dockerfile.worker b/devops/Dockerfile.worker index 28c26a15..f60bbb38 100644 --- a/devops/Dockerfile.worker +++ b/devops/Dockerfile.worker @@ -37,6 +37,8 @@ RUN apt-get update && apt-get upgrade -y --no-install-recommends \ libc6-dev \ libtbb-dev\ libcgal-dev \ + libsqlite3-dev \ + zlib1g-dev \ && rm -rf /var/lib/apt/lists/* WORKDIR /opt @@ -64,6 +66,16 @@ RUN sed -i '//r /tmp/nsf_logo_snippet.txt' /opt/PotreeConverter/build/reso # - remove reference to background image RUN sed -i 's/style="[^"]*background-image:[^"]*"//' /opt/PotreeConverter/build/resources/page_template/viewer_template.html +# Install tippecanoe +ARG TIPPECANOE_VERSION=2.79.0 +WORKDIR /opt +RUN git clone --depth 1 --branch "${TIPPECANOE_VERSION}" https://github.com/felt/tippecanoe.git \ + && cd tippecanoe \ + && make -j"$(nproc)" \ + && make install \ + && cd /opt \ + && rm -rf /opt/tippecanoe + # Copy pdal binary and libs from builder COPY --from=pdal-builder /opt/conda/envs/pdal /opt/pdal diff --git a/geoapi/services/tippecanoe.py b/geoapi/services/tippecanoe.py new file mode 100644 index 00000000..c7a13a89 --- /dev/null +++ b/geoapi/services/tippecanoe.py @@ -0,0 +1,78 @@ +import os +import subprocess + +from geoapi.log import logging + +logger = logging.getLogger(__name__) + +# tippecanoe binary; overridable for environments where it is not on PATH +TIPPECANOE_BIN = os.environ.get("TIPPECANOE_BIN", "tippecanoe") + + +class TippecanoeService: + """ + Utilities for converting GeoJSON into PMTiles vector tiles via tippecanoe. + + tippecanoe (https://github.com/felt/tippecanoe) is invoked as a subprocess. + """ + + @staticmethod + def geojson_to_pmtiles( + geojson_path: str, + output_path: str, + layer_name: str, + min_zoom: int = None, + max_zoom: int = None, + read_parallel: bool = False, + ) -> str: + """ + Convert a GeoJSON file into a PMTiles archive using tippecanoe. + + :param geojson_path: path to the source GeoJSON file + :param output_path: path where the .pmtiles archive will be written + :param layer_name: name of the vector tile layer + :param min_zoom: optional minimum zoom; defaults to tippecanoe's default + :param max_zoom: optional maximum zoom; when omitted, tippecanoe guesses + an appropriate maximum zoom via ``-zg`` + :param read_parallel: read the input in parallel (``-P``); only valid for + line-delimited GeoJSON input, so disabled by default + :return: output_path + :raises RuntimeError: if the tippecanoe binary is missing or exits non-zero + """ + cmd = [ + TIPPECANOE_BIN, + "-o", + output_path, + "--force", # overwrite output_path if it already exists + "-l", + layer_name, + "--drop-densest-as-needed", # shed features rather than overflow a tile + ] + if max_zoom is not None: + cmd += ["-z", str(max_zoom)] + else: + cmd += ["-zg"] # guess an appropriate maximum zoom + if min_zoom is not None: + cmd += ["-Z", str(min_zoom)] + if read_parallel: + cmd += ["-P"] + cmd.append(geojson_path) + + logger.info("Running tippecanoe: %s", " ".join(cmd)) + try: + result = subprocess.run(cmd, capture_output=True, text=True) + except FileNotFoundError as e: + raise RuntimeError( + f"tippecanoe binary not found (looked for '{TIPPECANOE_BIN}'). " + "Install tippecanoe or set the TIPPECANOE_BIN environment variable." + ) from e + + if result.returncode != 0: + logger.error( + "tippecanoe failed (exit %s): %s", result.returncode, result.stderr + ) + raise RuntimeError( + f"tippecanoe failed with exit code {result.returncode}: " + f"{result.stderr.strip()}" + ) + return output_path diff --git a/geoapi/tests/services/test_tippecanoe_service.py b/geoapi/tests/services/test_tippecanoe_service.py new file mode 100644 index 00000000..8c34057a --- /dev/null +++ b/geoapi/tests/services/test_tippecanoe_service.py @@ -0,0 +1,76 @@ +from unittest.mock import patch + +import pytest + +from geoapi.services.tippecanoe import TippecanoeService + + +def _completed(returncode=0, stderr=""): + class _Result: + pass + + result = _Result() + result.returncode = returncode + result.stderr = stderr + result.stdout = "" + return result + + +def test_geojson_to_pmtiles_builds_expected_command(): + with patch("geoapi.services.tippecanoe.subprocess.run") as mock_run: + mock_run.return_value = _completed() + out = TippecanoeService.geojson_to_pmtiles( + "/tmp/in.geojson", "/tmp/out.pmtiles", "my_layer" + ) + + assert out == "/tmp/out.pmtiles" + cmd = mock_run.call_args[0][0] + # output, force, layer, density management, guessed zoom, and input file + assert cmd[0] == "tippecanoe" + assert "-o" in cmd and cmd[cmd.index("-o") + 1] == "/tmp/out.pmtiles" + assert "--force" in cmd + assert "-l" in cmd and cmd[cmd.index("-l") + 1] == "my_layer" + assert "--drop-densest-as-needed" in cmd + assert "-zg" in cmd + assert cmd[-1] == "/tmp/in.geojson" + # parallel read is opt-in + assert "-P" not in cmd + + +def test_geojson_to_pmtiles_explicit_zoom_and_parallel(): + with patch("geoapi.services.tippecanoe.subprocess.run") as mock_run: + mock_run.return_value = _completed() + TippecanoeService.geojson_to_pmtiles( + "/tmp/in.geojson", + "/tmp/out.pmtiles", + "layer", + min_zoom=2, + max_zoom=14, + read_parallel=True, + ) + + cmd = mock_run.call_args[0][0] + assert "-z" in cmd and cmd[cmd.index("-z") + 1] == "14" + assert "-Z" in cmd and cmd[cmd.index("-Z") + 1] == "2" + assert "-P" in cmd + # explicit max zoom replaces the auto-guess flag + assert "-zg" not in cmd + + +def test_geojson_to_pmtiles_missing_binary_raises_runtimeerror(): + with patch( + "geoapi.services.tippecanoe.subprocess.run", side_effect=FileNotFoundError() + ): + with pytest.raises(RuntimeError, match="tippecanoe binary not found"): + TippecanoeService.geojson_to_pmtiles( + "/tmp/in.geojson", "/tmp/out.pmtiles", "layer" + ) + + +def test_geojson_to_pmtiles_nonzero_exit_raises_runtimeerror(): + with patch("geoapi.services.tippecanoe.subprocess.run") as mock_run: + mock_run.return_value = _completed(returncode=1, stderr="boom") + with pytest.raises(RuntimeError, match="exit code 1"): + TippecanoeService.geojson_to_pmtiles( + "/tmp/in.geojson", "/tmp/out.pmtiles", "layer" + ) From 8cf68cdb2dddde0542db89132a5904ab57ee7fdf Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 2 Jul 2026 09:55:17 -0500 Subject: [PATCH 02/23] Add VectorService.convert_to_geojson for vector file conversion --- geoapi/services/vectors.py | 106 +++++++++++++++++- .../tests/api_tests/test_vectors_service.py | 50 ++++++++- 2 files changed, 154 insertions(+), 2 deletions(-) diff --git a/geoapi/services/vectors.py b/geoapi/services/vectors.py index 4817b016..835ca7f9 100644 --- a/geoapi/services/vectors.py +++ b/geoapi/services/vectors.py @@ -1,11 +1,24 @@ import geopandas as gpd +from shapely import force_2d from geoapi.log import logging -from typing import List, IO +from typing import List, IO, Optional, Tuple +from pathlib import Path import tempfile import os logger = logging.getLogger(__name__) +# Vector file extensions that are ingested via convert_to_geojson() -> tippecanoe -> PMTiles +SUPPORTED_VECTOR_EXTENSIONS = { + "geojson", + "json", + "gpkg", + "parquet", + "geoparquet", + "shp", + "gpx", +} + # additional files for FILE.shp # (see https://desktop.arcgis.com/en/arcmap/10.3/manage-data/shapefiles/shapefile-file-extensions.htm) SHAPEFILE_FILE_ADDITIONAL_FILES = { @@ -61,3 +74,94 @@ def process_shapefile(shape_file: IO, additional_files: List[IO]): key: value for key, value in row.items() if key != "geometry" } yield row["geometry"], properties + + @staticmethod + def convert_to_geojson( + fileObj: IO, additional_files: Optional[List[IO]] = None + ) -> Tuple[str, dict]: + """Convert any supported vector file into a single EPSG:4326 GeoJSON file. + + Reads the file via geopandas, reprojects to EPSG:4326, strips Z + coordinates, and computes the bounding box. The GeoJSON is written into a + freshly-created temporary directory whose path is returned; the caller is + responsible for removing that directory when finished with it. + + :param fileObj: main vector file (must have ``.filename`` and ``.read()``) + :param additional_files: companion files (e.g. shapefile .dbf/.shx/.prj) + :return: ``(geojson_path, bbox)`` where bbox is a dict with keys + ``minx``, ``miny``, ``maxx``, ``maxy`` + """ + ext = Path(fileObj.filename).suffix.lstrip(".").lower() + if ext == "shp": + gdf = VectorService._read_shapefile(fileObj, additional_files or []) + elif ext == "gpx": + gdf = VectorService._read_gpx(fileObj) + else: + gdf = VectorService._read_direct(fileObj, ext) + + if gdf.crs is None: + # Inputs without a declared CRS (e.g. GeoJSON per RFC 7946) are WGS84 + gdf = gdf.set_crs(epsg=4326, allow_override=True) + else: + gdf = gdf.to_crs(epsg=4326) + + gdf = VectorService._force_2d(gdf) + + minx, miny, maxx, maxy = gdf.total_bounds + bbox = { + "minx": float(minx), + "miny": float(miny), + "maxx": float(maxx), + "maxy": float(maxy), + } + + out_dir = tempfile.mkdtemp(prefix="geoapi_vector_") + geojson_path = os.path.join(out_dir, "converted.geojson") + gdf.to_file(geojson_path, driver="GeoJSON") + return geojson_path, bbox + + @staticmethod + def _read_shapefile(shape_file: IO, additional_files: List[IO]) -> gpd.GeoDataFrame: + """Read a shapefile (with its companion files) into a GeoDataFrame.""" + all_files = list(additional_files) + all_files.append(shape_file) + with tempfile.TemporaryDirectory() as tmpdirname: + for f in all_files: + tmp_path = os.path.join(tmpdirname, os.path.basename(f.filename)) + with open(tmp_path, "wb") as tmp: + tmp.write(f.read()) + shapefile_path = os.path.join( + tmpdirname, os.path.basename(shape_file.filename) + ) + return gpd.read_file(shapefile_path) + + @staticmethod + def _read_gpx(fileObj: IO) -> gpd.GeoDataFrame: + """Read the tracks layer of a GPX file into a GeoDataFrame.""" + with tempfile.TemporaryDirectory() as tmpdirname: + tmp_path = os.path.join(tmpdirname, os.path.basename(fileObj.filename)) + with open(tmp_path, "wb") as tmp: + tmp.write(fileObj.read()) + return gpd.read_file(tmp_path, layer="tracks") + + @staticmethod + def _read_direct(fileObj: IO, ext: str) -> gpd.GeoDataFrame: + """Read a self-contained vector file (geojson/json/gpkg/parquet).""" + with tempfile.TemporaryDirectory() as tmpdirname: + tmp_path = os.path.join(tmpdirname, os.path.basename(fileObj.filename)) + with open(tmp_path, "wb") as tmp: + tmp.write(fileObj.read()) + if ext in ("parquet", "geoparquet"): + return gpd.read_parquet(tmp_path) + return gpd.read_file(tmp_path) + + @staticmethod + def _force_2d(gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame: + """Drop Z (and M) coordinates from every geometry in the GeoDataFrame.""" + if gdf.empty: + return gdf + gdf = gdf.copy() + gdf["geometry"] = gdf.geometry.apply( + lambda geom: force_2d(geom) if geom is not None else geom + ) + return gdf diff --git a/geoapi/tests/api_tests/test_vectors_service.py b/geoapi/tests/api_tests/test_vectors_service.py index 4a67706e..f93ec237 100644 --- a/geoapi/tests/api_tests/test_vectors_service.py +++ b/geoapi/tests/api_tests/test_vectors_service.py @@ -1,4 +1,10 @@ -from geoapi.services.vectors import VectorService +import os +import shutil + +import geopandas as gpd +from shapely.geometry import Point + +from geoapi.services.vectors import VectorService, SUPPORTED_VECTOR_EXTENSIONS import pyogrio import pytest @@ -29,3 +35,45 @@ def test_process_shapefile_missing_additional_files(shapefile_fixture): _, _ = next( VectorService.process_shapefile(shapefile_fixture, additional_files=[]) ) + + +def test_supported_vector_extensions(): + assert {"geojson", "shp", "gpx", "gpkg", "parquet", "geoparquet"}.issubset( + SUPPORTED_VECTOR_EXTENSIONS + ) + + +@pytest.mark.worker +def test_convert_to_geojson_shapefile( + shapefile_fixture, shapefile_additional_files_fixture +): + geojson_path, bbox = VectorService.convert_to_geojson( + shapefile_fixture, additional_files=shapefile_additional_files_fixture + ) + try: + # a GeoJSON file was written to a real temp directory + assert os.path.isfile(geojson_path) + + # bbox is a valid lon/lat extent + assert set(bbox) == {"minx", "miny", "maxx", "maxy"} + assert bbox["minx"] < bbox["maxx"] + assert bbox["miny"] < bbox["maxy"] + # within lon/lat range, allowing tiny FP slop at the antimeridian/poles + assert -180.01 <= bbox["minx"] and bbox["maxx"] <= 180.01 + assert -90.01 <= bbox["miny"] and bbox["maxy"] <= 90.01 + + # output is EPSG:4326 and 2D + gdf = gpd.read_file(geojson_path) + assert gdf.crs.to_epsg() == 4326 + assert not gdf.geometry.iloc[0].has_z + finally: + shutil.rmtree(os.path.dirname(geojson_path), ignore_errors=True) + + +@pytest.mark.worker +def test_force_2d_strips_z(): + gdf = gpd.GeoDataFrame(geometry=[Point(1.0, 2.0, 3.0)], crs="EPSG:4326") + assert gdf.geometry.iloc[0].has_z + + result = VectorService._force_2d(gdf) + assert not result.geometry.iloc[0].has_z From 697371c2af6e47461e3d57af703d160925cd542d Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 3 Jul 2026 11:18:11 -0500 Subject: [PATCH 03/23] Route vector file uploads through tippecanoe to PMTiles --- geoapi/routes/projects.py | 14 +- geoapi/services/features.py | 150 ++++++++---------- .../tests/api_tests/test_feature_service.py | 36 +++-- .../tests/api_tests/test_projects_routes.py | 8 +- .../external_data_tests/test_external_data.py | 14 +- geoapi/utils/features.py | 3 + 6 files changed, 120 insertions(+), 105 deletions(-) diff --git a/geoapi/routes/projects.py b/geoapi/routes/projects.py index a1fb15aa..9147a8c3 100644 --- a/geoapi/routes/projects.py +++ b/geoapi/routes/projects.py @@ -1,4 +1,5 @@ import io +import pathlib from datetime import datetime from typing import TYPE_CHECKING, Annotated from pydantic import BaseModel, Field @@ -10,12 +11,14 @@ from geojson_pydantic import Feature as GeoJSONFeature from geoapi.log import logger from geoapi.services.features import FeaturesService +from geoapi.services.vectors import SUPPORTED_VECTOR_EXTENSIONS from geoapi.services.streetview import StreetviewService from geoapi.services.point_cloud import PointCloudService from geoapi.services.projects import ProjectsService from geoapi.services.tile_server import TileService from geoapi.tasks import external_data, streetview, point_cloud from geoapi.models import Task, Project, Feature, TileServer, Overlay, PointCloud, User +from geoapi.exceptions import ApiException from geoapi.utils.decorators import ( project_permissions_allow_public_guard, project_permissions_guard, @@ -445,7 +448,9 @@ class ProjectFeaturesFilsResourceController(Controller): operation_id="upload_feature_file", description=""" Add a new feature(s) to a project from a file that has embedded geospatial information. - Current allowed file types are GeoJSON, georeferenced image (jpeg) or gpx track. + Current allowed file types are georeferenced image (jpeg). + Vector files (GeoJSON, GPX, shapefile, etc.) are not supported here; import them + from Tapis via POST /projects/{project_id}/features/files/import/ instead. Any additional key/value pairs in the form will also be placed in the feature(s) metadata""", guards=[project_permissions_guard], return_dto=FeatureReturnDTO, @@ -461,6 +466,13 @@ async def upload_feature_file( ) -> list[FeatureModel]: """Upload a file containing features to a project.""" file = data.pop("file") + ext = pathlib.Path(file.filename).suffix.lstrip(".").lower() + if ext in SUPPORTED_VECTOR_EXTENSIONS: + raise ApiException( + "Vector file uploads are not supported via direct HTTP upload. " + "Import the file from Tapis via " + "POST /projects/{project_id}/features/files/import/ instead." + ) file_bytes = await file.read() file_obj = io.BytesIO(file_bytes) file_obj.filename = file.filename diff --git a/geoapi/services/features.py b/geoapi/services/features.py index 62150924..93fc1139 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -5,17 +5,18 @@ import tempfile import configparser import re +import shutil from typing import List, IO, Dict from geoapi.services.tile_server import TileService from geoapi.services.videos import VideoService -from shapely.geometry import Point, shape -import fiona +from shapely.geometry import Point, box from geoalchemy2.shape import from_shape import geojson from geoapi.services.images import ImageService, ImageData -from geoapi.services.vectors import VectorService +from geoapi.services.tippecanoe import TippecanoeService +from geoapi.services.vectors import VectorService, SUPPORTED_VECTOR_EXTENSIONS from geoapi.models import Feature, FeatureAsset, Overlay, User, TileServer from geoapi.exceptions import ( InvalidGeoJSON, @@ -29,7 +30,7 @@ get_asset_relative_path, ) from geoapi.log import logging -from geoapi.utils import geometries, features as features_util +from geoapi.utils import features as features_util from geoapi.utils.external_apis import TapisUtils from geoapi.utils.geo_location import GeoLocation, parse_rapid_geolocation @@ -170,87 +171,75 @@ def fromLatLng( return f @staticmethod - def fromGPX( - database_session, - projectId: int, - fileObj: IO, - metadata: Dict, - original_system, # ignored - original_path, # ignored - ) -> Feature: - - # TODO: Fiona should support reading from the file directly, this MemoryFile business - # should not be needed - with fiona.io.MemoryFile(fileObj) as memfile: - with memfile.open(layer="tracks") as collection: - track = collection[0] - shp = shape(track["geometry"]) - feat = Feature() - feat.project_id = projectId - feat.the_geom = from_shape(geometries.convert_3D_2D(shp), srid=4326) - feat.properties = metadata or {} - # TODO original_path, original_system are ignored but should not be ignored after WG-600 - - database_session.add(feat) - database_session.commit() - return feat - - @staticmethod - def fromGeoJSON( + def fromVectorFile( database_session, projectId: int, fileObj: IO, metadata: Dict, original_system: str = None, original_path: str = None, - ) -> List[Feature]: - """ + additional_files: List[IO] = None, + ) -> Feature: + """Create a single Feature backed by a PMTiles vector asset. + + The vector file is converted to EPSG:4326 GeoJSON, run through tippecanoe + to produce a ``.pmtiles`` archive, and stored as a FeatureAsset. The + Feature's geometry is the bounding-box polygon of the source data. :param projectId: int - :param fileObj: file descriptor - :param metadata: Dict of pairs - :param original_path: str path of original file location - :return: Feature + :param fileObj: main vector file + :param metadata: Dict of pairs stored on the feature + :param additional_files: companion files (e.g. shapefile .dbf/.shx/.prj) + :return: the created Feature """ - data = json.loads(fileObj.read()) - fileObj.close() - return FeaturesService.addGeoJSON( - database_session, projectId, data, original_system, original_path - ) + geojson_dir = None + try: + geojson_path, bbox = VectorService.convert_to_geojson( + fileObj, additional_files + ) + geojson_dir = os.path.dirname(geojson_path) - @staticmethod - def fromShapefile( - database_session, - projectId: int, - fileObj: IO, - metadata: Dict, - additional_files: List[IO], - original_system=None, - original_path=None, - ) -> Feature: - """Create features from shapefile + # layer name derived from the filename stem, sanitized for tippecanoe + stem = pathlib.Path(fileObj.filename).stem + layer_name = re.sub(r"[^a-zA-Z0-9_-]", "_", stem) or "layer" - :param projectId: int - :param fileObj: file descriptor - :param additional_files: file descriptor for all the other non-.shp files - :param metadata: Dict of pairs [IGNORED} - :param original_path: str path of original file location [IGNORED} - :return: Feature - """ - features = [] - for geom, properties in VectorService.process_shapefile( - fileObj, additional_files - ): + asset_uuid = uuid.uuid4() + base_filepath = make_project_asset_dir(projectId) + asset_path = os.path.join(base_filepath, str(asset_uuid) + ".pmtiles") + + # build the pmtiles in a temp dir, then move it into the asset dir + with tempfile.TemporaryDirectory() as tmp_pmtiles_dir: + tmp_pmtiles_path = os.path.join(tmp_pmtiles_dir, "output.pmtiles") + TippecanoeService.geojson_to_pmtiles( + geojson_path, tmp_pmtiles_path, layer_name + ) + shutil.copyfile(tmp_pmtiles_path, asset_path) + + bbox_geom = box(bbox["minx"], bbox["miny"], bbox["maxx"], bbox["maxy"]) feat = Feature() feat.project_id = projectId - feat.the_geom = from_shape(geometries.convert_3D_2D(geom), srid=4326) - feat.properties = properties + feat.the_geom = from_shape(bbox_geom, srid=4326) + feat.properties = metadata or {} # TODO original_path, original_system are ignored but should not be ignored after WG-600 - database_session.add(feat) - features.append(feat) - database_session.commit() - return features + fa = FeatureAsset( + uuid=asset_uuid, + asset_type="vector", + original_system=original_system, + original_path=original_path, + display_path=original_path, + path=get_asset_relative_path(asset_path), + feature=feat, + ) + feat.assets.append(fa) + + database_session.add(feat) + database_session.commit() + return feat + finally: + if geojson_dir and os.path.isdir(geojson_dir): + shutil.rmtree(geojson_dir, ignore_errors=True) + fileObj.close() @staticmethod def from_rapp_questionnaire( @@ -444,31 +433,18 @@ def fromFileObj( location, ) ] - elif ext in features_util.GPX_FILE_EXTENSIONS: + elif ext in SUPPORTED_VECTOR_EXTENSIONS: return [ - FeaturesService.fromGPX( + FeaturesService.fromVectorFile( database_session, projectId, fileObj, - metadata, + {}, original_system, original_path, + additional_files, ) ] - elif ext in features_util.GEOJSON_FILE_EXTENSIONS: - return FeaturesService.fromGeoJSON( - database_session, projectId, fileObj, {}, original_system, original_path - ) - elif ext in features_util.SHAPEFILE_FILE_EXTENSIONS: - return FeaturesService.fromShapefile( - database_session, - projectId, - fileObj, - {}, - additional_files, - original_system, - original_path, - ) elif ext in features_util.INI_FILE_EXTENSIONS: return FeaturesService.fromINI(database_session, projectId, fileObj, {}) elif ext in features_util.RAPP_QUESTIONNAIRE_FILE_EXTENSIONS: diff --git a/geoapi/tests/api_tests/test_feature_service.py b/geoapi/tests/api_tests/test_feature_service.py index 70536936..47fe9216 100644 --- a/geoapi/tests/api_tests/test_feature_service.py +++ b/geoapi/tests/api_tests/test_feature_service.py @@ -1,5 +1,7 @@ +import json import os +import pytest from werkzeug.datastructures import FileStorage from geoapi.services.features import FeaturesService from geoapi.models import Feature, FeatureAsset @@ -19,8 +21,10 @@ def test_create_feature_fromLatLng(projects_fixture, db_session): def test_hazmapperv1_file_with_images(projects_fixture, hazmpperV1_file, db_session): - features = FeaturesService.fromGeoJSON( - db_session, projects_fixture.id, hazmpperV1_file, metadata={} + # container features from a v1 Hazmapper GeoJSON go through addGeoJSON (Path A), + # which extracts any embedded base64 images into assets + features = FeaturesService.addGeoJSON( + db_session, projects_fixture.id, json.loads(hazmpperV1_file.read()) ) assert len(features) == 2 assert len(features[1].assets) == 1 @@ -29,8 +33,10 @@ def test_hazmapperv1_file_with_images(projects_fixture, hazmpperV1_file, db_sess def test_insert_feature_geojson( projects_fixture, feature_properties_file_fixture, db_session ): - features = FeaturesService.fromGeoJSON( - db_session, projects_fixture.id, feature_properties_file_fixture, metadata={} + features = FeaturesService.addGeoJSON( + db_session, + projects_fixture.id, + json.loads(feature_properties_file_fixture.read()), ) feature = features[0] assert len(features) == 1 @@ -41,8 +47,8 @@ def test_insert_feature_geojson( def test_insert_feature_collection(projects_fixture, geojson_file_fixture, db_session): - features = FeaturesService.fromGeoJSON( - db_session, projects_fixture.id, geojson_file_fixture, metadata={} + features = FeaturesService.addGeoJSON( + db_session, projects_fixture.id, json.loads(geojson_file_fixture.read()) ) for feature in features: assert feature.project_id == projects_fixture.id @@ -194,21 +200,29 @@ def test_remove_feature_video_asset( assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 0 +@pytest.mark.worker def test_create_feature_shpfile( projects_fixture, shapefile_fixture, shapefile_additional_files_fixture, db_session ): - features = FeaturesService.fromShapefile( + feature = FeaturesService.fromVectorFile( db_session, projects_fixture.id, shapefile_fixture, metadata={}, additional_files=shapefile_additional_files_fixture, + original_system="system", original_path="foo", ) - assert len(features) == 10 - assert db_session.query(Feature).count() == 10 - assert features[0].project_id == projects_fixture.id - # TODO Test original_path, original_system are configured after WG-600 + # a vector file is ingested as a single bbox Feature backed by a PMTiles asset + assert feature.project_id == projects_fixture.id + assert db_session.query(Feature).count() == 1 + assert len(feature.assets) == 1 + asset = feature.assets[0] + assert asset.asset_type == "vector" + assert os.path.isfile(get_asset_path(asset.path)) + assert asset.original_system == "system" + assert asset.original_path == "foo" + assert asset.display_path == "foo" def test_create_questionnaire_feature( diff --git a/geoapi/tests/api_tests/test_projects_routes.py b/geoapi/tests/api_tests/test_projects_routes.py index 5f15a1c3..db313483 100644 --- a/geoapi/tests/api_tests/test_projects_routes.py +++ b/geoapi/tests/api_tests/test_projects_routes.py @@ -287,13 +287,17 @@ def test_delete_user_unauthorized(test_client, projects_fixture, user2): assert resp.status_code == 403 -def test_upload_gpx(test_client, projects_fixture, gpx_file_fixture, user1): +def test_upload_gpx_vector_rejected( + test_client, projects_fixture, gpx_file_fixture, user1 +): + # vector files are no longer processed via the synchronous HTTP upload route; + # they must be imported from Tapis resp = test_client.post( f"/projects/{projects_fixture.id}/features/files/", files={"file": gpx_file_fixture}, headers={"X-Tapis-Token": user1.jwt}, ) - assert resp.status_code == 201 + assert resp.status_code == 400 def test_upload_image(test_client, projects_fixture, image_file_fixture, user1): diff --git a/geoapi/tests/external_data_tests/test_external_data.py b/geoapi/tests/external_data_tests/test_external_data.py index beb70114..38015b68 100644 --- a/geoapi/tests/external_data_tests/test_external_data.py +++ b/geoapi/tests/external_data_tests/test_external_data.py @@ -222,8 +222,13 @@ def test_external_data_good_files( projects_fixture.id, ) features = db_session.query(Feature).all() - # the test geojson has 3 features in it - assert len(features) == 3 + # the geojson is now ingested as a single bbox Feature backed by a PMTiles + # asset produced by tippecanoe (which runs for real in the worker image) + assert len(features) == 1 + assert len(features[0].assets) == 1 + asset = features[0].assets[0] + assert asset.asset_type == "vector" + assert os.path.exists(get_asset_path(asset.path)) imported_file = db_session.query(ImportedFile).first() assert imported_file.successful_import @@ -636,8 +641,9 @@ def test_refresh_projects_watch_content( assert "rollback" not in caplog.text features = db_session.query(Feature).all() - # the test geojson has 3 features in it - assert len(features) == 3 + # the geojson is ingested as a single bbox Feature backed by a PMTiles asset + assert len(features) == 1 + assert features[0].assets[0].asset_type == "vector" @pytest.mark.worker diff --git a/geoapi/utils/features.py b/geoapi/utils/features.py index fa19b28f..7f92ee4d 100644 --- a/geoapi/utils/features.py +++ b/geoapi/utils/features.py @@ -16,6 +16,8 @@ SHAPEFILE_FILE_EXTENSIONS = ("shp",) +ADDITIONAL_VECTOR_FILE_EXTENSIONS = ("gpkg", "parquet", "geoparquet") + RAPP_QUESTIONNAIRE_FILE_EXTENSIONS = ("rq",) RAPP_QUESTIONNAIRE_ARCHIVE_EXTENSIONS = "rqa" @@ -32,6 +34,7 @@ + GPX_FILE_EXTENSIONS + GEOJSON_FILE_EXTENSIONS + SHAPEFILE_FILE_EXTENSIONS + + ADDITIONAL_VECTOR_FILE_EXTENSIONS + RAPP_QUESTIONNAIRE_FILE_EXTENSIONS ) From 26cf1308f984ae3d861a4fe6c5277a9186267b4d Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 3 Jul 2026 11:39:37 -0500 Subject: [PATCH 04/23] Remove orphaned process_shapefile --- geoapi/services/vectors.py | 31 ---------------- .../tests/api_tests/test_vectors_service.py | 35 ++++--------------- 2 files changed, 7 insertions(+), 59 deletions(-) diff --git a/geoapi/services/vectors.py b/geoapi/services/vectors.py index 835ca7f9..96fba5a1 100644 --- a/geoapi/services/vectors.py +++ b/geoapi/services/vectors.py @@ -44,37 +44,6 @@ class VectorService: Utilities for handling vector files """ - @staticmethod - def process_shapefile(shape_file: IO, additional_files: List[IO]): - """Process shapefile - - Loads shapefile and converts it to epsg 4326 - - :param shape_file: IO - :param additional_files: List[IO] other files needed besides the main .shp file - :return: generator that provides geometry plus properties for each item - """ - all_files = additional_files.copy() - all_files.append(shape_file) - - with tempfile.TemporaryDirectory() as tmpdirname: - # save files together for processing purposes - for f in all_files: - tmp_path = os.path.join(tmpdirname, os.path.basename(f.filename)) - with open(tmp_path, "wb") as tmp: - tmp.write(f.read()) - - shapefile_path = os.path.join( - tmpdirname, os.path.basename(shape_file.filename) - ) - shapefile = gpd.read_file(shapefile_path) - shapefile = shapefile.to_crs(epsg=4326) - for index, row in shapefile.iterrows(): - properties = { - key: value for key, value in row.items() if key != "geometry" - } - yield row["geometry"], properties - @staticmethod def convert_to_geojson( fileObj: IO, additional_files: Optional[List[IO]] = None diff --git a/geoapi/tests/api_tests/test_vectors_service.py b/geoapi/tests/api_tests/test_vectors_service.py index f93ec237..278d9f75 100644 --- a/geoapi/tests/api_tests/test_vectors_service.py +++ b/geoapi/tests/api_tests/test_vectors_service.py @@ -9,34 +9,6 @@ import pytest -def test_process_shapefile( - shapefile_fixture, - shapefile_additional_files_fixture, - shapefile_first_element_geometry, -): - geom, properties = next( - VectorService.process_shapefile( - shapefile_fixture, additional_files=shapefile_additional_files_fixture - ) - ) - - assert geom.wkt == shapefile_first_element_geometry - assert properties == { - "continent": "South America", - "gdp_md_est": 436100.0, - "iso_a3": "CHL", - "name": "Chile", - "pop_est": 17789267, - } - - -def test_process_shapefile_missing_additional_files(shapefile_fixture): - with pytest.raises(pyogrio.errors.DataSourceError): - _, _ = next( - VectorService.process_shapefile(shapefile_fixture, additional_files=[]) - ) - - def test_supported_vector_extensions(): assert {"geojson", "shp", "gpx", "gpkg", "parquet", "geoparquet"}.issubset( SUPPORTED_VECTOR_EXTENSIONS @@ -70,6 +42,13 @@ def test_convert_to_geojson_shapefile( shutil.rmtree(os.path.dirname(geojson_path), ignore_errors=True) +@pytest.mark.worker +def test_convert_to_geojson_missing_shapefile_additional_files(shapefile_fixture): + # a shapefile cannot be read without its companion files (.shx/.dbf/...) + with pytest.raises(pyogrio.errors.DataSourceError): + VectorService.convert_to_geojson(shapefile_fixture, additional_files=[]) + + @pytest.mark.worker def test_force_2d_strips_z(): gdf = gpd.GeoDataFrame(geometry=[Point(1.0, 2.0, 3.0)], crs="EPSG:4326") From 5bff97da55d6906ba2feedb02fcb49d3daf02d3b Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Mon, 6 Jul 2026 18:11:39 -0500 Subject: [PATCH 05/23] Fix vector PMTiles rendering end-to-end - Floor tippecanoe's guessed max zoom so sparse/global vector data still produces renderable tiles (and simplify geojson_to_pmtiles to its used args) - Include vector-asset features in the no_asset_vector feature query - List the containing directory when importing so only existing companion shapefile files are fetched (avoids noisy 404s) - Serve .pmtiles assets with byte-range support and CORS preflight in nginx - Add a decode-and-inspect test plus a point+polygon fixture --- devops/local_conf/nginx.conf | 21 +++++++++ geoapi/services/projects.py | 4 +- geoapi/services/tippecanoe.py | 38 ++++++++------- geoapi/tasks/external_data.py | 25 +++++++++- .../tests/api_tests/test_vectors_service.py | 33 +++++++++++++ geoapi/tests/conftest.py | 9 ++++ .../fixtures/TACC_point_and_polygon.geojson | 47 +++++++++++++++++++ .../tests/services/test_tippecanoe_service.py | 22 +-------- 8 files changed, 159 insertions(+), 40 deletions(-) create mode 100644 geoapi/tests/fixtures/TACC_point_and_polygon.geojson diff --git a/devops/local_conf/nginx.conf b/devops/local_conf/nginx.conf index a9766488..45632bac 100644 --- a/devops/local_conf/nginx.conf +++ b/devops/local_conf/nginx.conf @@ -71,6 +71,27 @@ http { gzip off; } + # PMTiles vector assets are read by the browser via cross-origin + # HTTP range requests, so we must: + # * allow byte ranges and disable gzip (gzip breaks range reads) + # * answer the CORS preflight, advertising the x-tapis-token + # header the app sends to authenticate the request + location ~ \.pmtiles$ { + if ($request_method = OPTIONS) { + add_header "Access-Control-Allow-Origin" * always; + add_header "Access-Control-Allow-Methods" "GET, HEAD, OPTIONS" always; + add_header "Access-Control-Allow-Headers" * always; + add_header "Access-Control-Max-Age" "86400" always; + add_header "Content-Length" "0" always; + return 204; + } + + add_header Accept-Ranges bytes; + add_header "Access-Control-Allow-Origin" * always; + add_header "Access-Control-Allow-Headers" * always; + gzip off; + } + # Preflighted requests if ($request_method = OPTIONS) { add_header "Access-Control-Allow-Origin" "*" always; diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index 831ff683..587f79a1 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -225,7 +225,9 @@ def getFeatures(database_session, projectId: int, query: dict = None) -> object: if asset: params[asset] = asset assetQueries.append( - "fa is null" + # "no_asset_vector" covers both features with no asset and + # features backed by a PMTiles vector asset + "(fa is null OR fa.asset_type = 'vector')" if asset == "no_asset_vector" else "fa.asset_type = :" + asset ) diff --git a/geoapi/services/tippecanoe.py b/geoapi/services/tippecanoe.py index c7a13a89..a0c83c62 100644 --- a/geoapi/services/tippecanoe.py +++ b/geoapi/services/tippecanoe.py @@ -8,6 +8,22 @@ # tippecanoe binary; overridable for environments where it is not on PATH TIPPECANOE_BIN = os.environ.get("TIPPECANOE_BIN", "tippecanoe") +# Floor for tippecanoe's guessed maximum zoom (-zg). +# +# `-zg` picks a max zoom from how far apart / how detailed the features are. +# For data whose features are simple and spread across a very large extent +# (e.g. a handful of points scattered globally, or continent-scale polygons) +# it can guess a max zoom of 0 — a single world tile. That is a problem for us: +# * positions are then quantized to a ~10km grid (useless when zoomed in), and +# * protomaps-leaflet cannot overzoom a zoom-0 archive (it treats +# maxDataZoom 0 as falsy and defaults to 15, requesting tiles that don't +# exist), so nothing renders. +# Flooring the guess to 12 (~sub-meter precision at the deepest tile) fixes both. +# The floor only *raises* low guesses; data with real local detail already +# guesses higher (e.g. dense reconnaissance points guess ~16) and is left +# untouched. +SMALLEST_MAXIMUM_ZOOM_GUESS = 12 + class TippecanoeService: """ @@ -21,9 +37,6 @@ def geojson_to_pmtiles( geojson_path: str, output_path: str, layer_name: str, - min_zoom: int = None, - max_zoom: int = None, - read_parallel: bool = False, ) -> str: """ Convert a GeoJSON file into a PMTiles archive using tippecanoe. @@ -31,11 +44,6 @@ def geojson_to_pmtiles( :param geojson_path: path to the source GeoJSON file :param output_path: path where the .pmtiles archive will be written :param layer_name: name of the vector tile layer - :param min_zoom: optional minimum zoom; defaults to tippecanoe's default - :param max_zoom: optional maximum zoom; when omitted, tippecanoe guesses - an appropriate maximum zoom via ``-zg`` - :param read_parallel: read the input in parallel (``-P``); only valid for - line-delimited GeoJSON input, so disabled by default :return: output_path :raises RuntimeError: if the tippecanoe binary is missing or exits non-zero """ @@ -47,16 +55,12 @@ def geojson_to_pmtiles( "-l", layer_name, "--drop-densest-as-needed", # shed features rather than overflow a tile + # guess an appropriate maximum zoom, but never below our floor + # (see SMALLEST_MAXIMUM_ZOOM_GUESS for why sparse global data needs it) + "-zg", + f"--smallest-maximum-zoom-guess={SMALLEST_MAXIMUM_ZOOM_GUESS}", + geojson_path, ] - if max_zoom is not None: - cmd += ["-z", str(max_zoom)] - else: - cmd += ["-zg"] # guess an appropriate maximum zoom - if min_zoom is not None: - cmd += ["-Z", str(min_zoom)] - if read_parallel: - cmd += ["-P"] - cmd.append(geojson_path) logger.info("Running tippecanoe: %s", " ".join(cmd)) try: diff --git a/geoapi/tasks/external_data.py b/geoapi/tasks/external_data.py index 70444149..58823bc9 100644 --- a/geoapi/tasks/external_data.py +++ b/geoapi/tasks/external_data.py @@ -145,7 +145,7 @@ def get_additional_files( f"Required file ({system_id}/{additional_file_path}) missing" ) if not result_file: - logger.error( + logger.warning( f"Unable to get non-required {file_suffix}-related file: " f"tapis: {system_id} :: {additional_file_path} ---- error: {error}" ) @@ -174,7 +174,28 @@ def import_file_from_tapis(userId: int, systemId: str, path: str, projectId: int client = TapisUtils(session, user) temp_file = client.getFile(systemId, path) temp_file.filename = Path(path).name - additional_files = get_additional_files(temp_file, systemId, path, client) + + # List the containing directory so get_additional_files only fetches + # companion files that actually exist (avoids 404s on optional + # shapefile files). + available_files = None + try: + parent = Path(path).parent + directory_listing = client.listing(systemId, str(parent)) + available_files = [ + str(parent / Path(item.path).name) for item in directory_listing + ] + except Exception: + logger.warning( + "Could not list directory for %s/%s; will attempt all " + "companion files", + systemId, + path, + ) + + additional_files = get_additional_files( + temp_file, systemId, path, client, available_files=available_files + ) optional_location_from_metadata = get_geolocation_from_file_metadata( session, user, system_id=systemId, path=path diff --git a/geoapi/tests/api_tests/test_vectors_service.py b/geoapi/tests/api_tests/test_vectors_service.py index 278d9f75..4e663f33 100644 --- a/geoapi/tests/api_tests/test_vectors_service.py +++ b/geoapi/tests/api_tests/test_vectors_service.py @@ -1,10 +1,13 @@ import os import shutil +import subprocess +import tempfile import geopandas as gpd from shapely.geometry import Point from geoapi.services.vectors import VectorService, SUPPORTED_VECTOR_EXTENSIONS +from geoapi.services.tippecanoe import TippecanoeService import pyogrio import pytest @@ -49,6 +52,36 @@ def test_convert_to_geojson_missing_shapefile_additional_files(shapefile_fixture VectorService.convert_to_geojson(shapefile_fixture, additional_files=[]) +@pytest.mark.worker +def test_point_and_polygon_geojson_tiles_retain_geometry( + point_and_polygon_geojson_fixture, +): + # Convert -> tippecanoe -> decode the tiles and confirm both the point and + # the polygon survive the pipeline (regression for mixed-geometry uploads). + geojson_path, _ = VectorService.convert_to_geojson( + point_and_polygon_geojson_fixture + ) + out_dir = tempfile.mkdtemp(prefix="geoapi_pmtiles_test_") + try: + pmtiles_path = os.path.join(out_dir, "out.pmtiles") + TippecanoeService.geojson_to_pmtiles(geojson_path, pmtiles_path, "tacc") + assert os.path.isfile(pmtiles_path) + + decoded = subprocess.run( + ["tippecanoe-decode", pmtiles_path], + capture_output=True, + text=True, + check=True, + ).stdout + + assert '"Point"' in decoded, "point geometry did not survive tiling" + assert '"Polygon"' in decoded, "polygon geometry did not survive tiling" + assert "tacc" in decoded, "expected vector layer 'tacc' not found in tiles" + finally: + shutil.rmtree(out_dir, ignore_errors=True) + shutil.rmtree(os.path.dirname(geojson_path), ignore_errors=True) + + @pytest.mark.worker def test_force_2d_strips_z(): gdf = gpd.GeoDataFrame(geometry=[Point(1.0, 2.0, 3.0)], crs="EPSG:4326") diff --git a/geoapi/tests/conftest.py b/geoapi/tests/conftest.py index 17e232a0..fb1d1996 100644 --- a/geoapi/tests/conftest.py +++ b/geoapi/tests/conftest.py @@ -430,6 +430,15 @@ def shapefile_fixture(): yield FileStorage(f) +@pytest.fixture(scope="function") +def point_and_polygon_geojson_fixture(): + home = os.path.dirname(__file__) + with open( + os.path.join(home, "fixtures/TACC_point_and_polygon.geojson"), "rb" + ) as f: + yield FileStorage(f, filename="TACC_point_and_polygon.geojson") + + @pytest.fixture(scope="function") def shapefile_additional_files_fixture(): home = os.path.dirname(__file__) diff --git a/geoapi/tests/fixtures/TACC_point_and_polygon.geojson b/geoapi/tests/fixtures/TACC_point_and_polygon.geojson new file mode 100644 index 00000000..997f6e33 --- /dev/null +++ b/geoapi/tests/fixtures/TACC_point_and_polygon.geojson @@ -0,0 +1,47 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + -97.72566326733316, + 30.390237629525316 + ], + "type": "Point" + } + }, + { + "type": "Feature", + "properties": {}, + "geometry": { + "coordinates": [ + [ + [ + -97.7264790213757, + 30.39002342986133 + ], + [ + -97.7264790213757, + 30.38977316915799 + ], + [ + -97.72578744974246, + 30.38977316915799 + ], + [ + -97.72578744974246, + 30.39002342986133 + ], + [ + -97.7264790213757, + 30.39002342986133 + ] + ] + ], + "type": "Polygon" + } + } + ] +} diff --git a/geoapi/tests/services/test_tippecanoe_service.py b/geoapi/tests/services/test_tippecanoe_service.py index 8c34057a..e83d3963 100644 --- a/geoapi/tests/services/test_tippecanoe_service.py +++ b/geoapi/tests/services/test_tippecanoe_service.py @@ -32,31 +32,13 @@ def test_geojson_to_pmtiles_builds_expected_command(): assert "-l" in cmd and cmd[cmd.index("-l") + 1] == "my_layer" assert "--drop-densest-as-needed" in cmd assert "-zg" in cmd + # the guessed max zoom is floored so sparse global data still renders + assert "--smallest-maximum-zoom-guess=12" in cmd assert cmd[-1] == "/tmp/in.geojson" # parallel read is opt-in assert "-P" not in cmd -def test_geojson_to_pmtiles_explicit_zoom_and_parallel(): - with patch("geoapi.services.tippecanoe.subprocess.run") as mock_run: - mock_run.return_value = _completed() - TippecanoeService.geojson_to_pmtiles( - "/tmp/in.geojson", - "/tmp/out.pmtiles", - "layer", - min_zoom=2, - max_zoom=14, - read_parallel=True, - ) - - cmd = mock_run.call_args[0][0] - assert "-z" in cmd and cmd[cmd.index("-z") + 1] == "14" - assert "-Z" in cmd and cmd[cmd.index("-Z") + 1] == "2" - assert "-P" in cmd - # explicit max zoom replaces the auto-guess flag - assert "-zg" not in cmd - - def test_geojson_to_pmtiles_missing_binary_raises_runtimeerror(): with patch( "geoapi.services.tippecanoe.subprocess.run", side_effect=FileNotFoundError() From f860185374ed9f59e7db24b2820b7c341b93827d Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 7 Jul 2026 12:15:01 -0500 Subject: [PATCH 06/23] Fix comment --- geoapi/tests/external_data_tests/test_external_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/geoapi/tests/external_data_tests/test_external_data.py b/geoapi/tests/external_data_tests/test_external_data.py index 38015b68..204ec4f1 100644 --- a/geoapi/tests/external_data_tests/test_external_data.py +++ b/geoapi/tests/external_data_tests/test_external_data.py @@ -222,8 +222,7 @@ def test_external_data_good_files( projects_fixture.id, ) features = db_session.query(Feature).all() - # the geojson is now ingested as a single bbox Feature backed by a PMTiles - # asset produced by tippecanoe (which runs for real in the worker image) + # the geojson is ingested as a single bbox Feature backed by a PMTiles assert len(features) == 1 assert len(features[0].assets) == 1 asset = features[0].assets[0] From c2f6272acedc39605d5cc3564e23593568c1c671 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 7 Jul 2026 12:15:31 -0500 Subject: [PATCH 07/23] Change tippecanoe params and log output --- geoapi/services/tippecanoe.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/geoapi/services/tippecanoe.py b/geoapi/services/tippecanoe.py index a0c83c62..e4e84b4f 100644 --- a/geoapi/services/tippecanoe.py +++ b/geoapi/services/tippecanoe.py @@ -54,10 +54,15 @@ def geojson_to_pmtiles( "--force", # overwrite output_path if it already exists "-l", layer_name, - "--drop-densest-as-needed", # shed features rather than overflow a tile - # guess an appropriate maximum zoom, but never below our floor - # (see SMALLEST_MAXIMUM_ZOOM_GUESS for why sparse global data needs it) + # prefer adding zoom levels over dropping features (data fidelity); + # runaway builds are bounded by the Celery task timeout + "--extend-zooms-if-still-dropping", + # shed features rather than overflow a tile + "--drop-densest-as-needed", + # guess maxzoom from feature spacing, but floor it so sparse/global + # datasets don't get a near-zero maxzoom "-zg", + # See SMALLEST_MAXIMUM_ZOOM_GUESS discussion above f"--smallest-maximum-zoom-guess={SMALLEST_MAXIMUM_ZOOM_GUESS}", geojson_path, ] @@ -79,4 +84,6 @@ def geojson_to_pmtiles( f"tippecanoe failed with exit code {result.returncode}: " f"{result.stderr.strip()}" ) + if result.stderr: + logger.debug("tippecanoe output: %s", result.stderr) return output_path From 434541672318f4ae7e814760b16f06c1736d3979 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 7 Jul 2026 16:15:41 -0500 Subject: [PATCH 08/23] Keep vector features visible at all zoom levels tippecanoe drops features at low zoom by default, leaving a scattered point layer nearly empty at the world view. Add --drop-rate=1 so every feature is kept at every zoom (--drop-densest-as-needed still caps oversized tiles), plus a worker test asserting >900 of the 1000 points survive at each zoom. --- geoapi/services/tippecanoe.py | 6 ++- geoapi/tests/conftest.py | 7 +++ geoapi/tests/fixtures/1000_points.geojson | 1 + .../tests/services/test_tippecanoe_service.py | 48 +++++++++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 geoapi/tests/fixtures/1000_points.geojson diff --git a/geoapi/services/tippecanoe.py b/geoapi/services/tippecanoe.py index e4e84b4f..a9f75a59 100644 --- a/geoapi/services/tippecanoe.py +++ b/geoapi/services/tippecanoe.py @@ -57,7 +57,11 @@ def geojson_to_pmtiles( # prefer adding zoom levels over dropping features (data fidelity); # runaway builds are bounded by the Celery task timeout "--extend-zooms-if-still-dropping", - # shed features rather than overflow a tile + # keep every feature at every zoom (no thinning as you zoom out); by + # default tippecanoe drops features at low zoom, so a scattered point + # layer would show only ~1 point at the world view + "--drop-rate=1", + # shed features only if a tile would otherwise overflow "--drop-densest-as-needed", # guess maxzoom from feature spacing, but floor it so sparse/global # datasets don't get a near-zero maxzoom diff --git a/geoapi/tests/conftest.py b/geoapi/tests/conftest.py index fb1d1996..777b9f12 100644 --- a/geoapi/tests/conftest.py +++ b/geoapi/tests/conftest.py @@ -439,6 +439,13 @@ def point_and_polygon_geojson_fixture(): yield FileStorage(f, filename="TACC_point_and_polygon.geojson") +@pytest.fixture(scope="function") +def points_1000_geojson_path_fixture(): + """Path to a GeoJSON of 1000 points scattered globally (EPSG:4326).""" + home = os.path.dirname(__file__) + return os.path.join(home, "fixtures/1000_points.geojson") + + @pytest.fixture(scope="function") def shapefile_additional_files_fixture(): home = os.path.dirname(__file__) diff --git a/geoapi/tests/fixtures/1000_points.geojson b/geoapi/tests/fixtures/1000_points.geojson new file mode 100644 index 00000000..37e4a218 --- /dev/null +++ b/geoapi/tests/fixtures/1000_points.geojson @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[-33.59323687792417,46.40696114862103]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[132.85442396224636,39.06042348211311]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-51.74847228439112,88.01842836752458]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-62.17815569595491,-81.66138456037758]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-157.32815488365588,70.4536914798808]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[108.48381998587647,-19.827805443518596]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[141.01326763910373,-33.055375635817256]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[2.6078885868934787,-62.7958567039361]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[111.9209608600534,34.33816233142244]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[152.9128894931918,-50.37848586401796]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[79.4970884890215,-21.599917576827174]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[25.17089146137918,-18.650845842297162]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[148.80157962071175,25.49877686572301]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[34.033644609478614,62.11036687094209]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[91.51141429536608,88.54538211543856]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[92.04405743304021,-54.96238835374544]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[146.598476723214,53.97428689923701]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-35.41531883973809,64.59378242720705]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[157.21278147756092,-60.17029268368439]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-64.92638483498676,-47.87180361431196]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-137.46924363035168,-82.69488173110523]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-141.72358731594883,-47.320986714201155]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-57.4580909656144,20.97212709895882]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-79.11698833788377,58.04399088032451]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-98.66861305988473,11.25702699806579]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[38.052486359979696,19.423625666135045]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[21.62498026064659,-55.48608223715044]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[100.53732516803366,62.76496090508046]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[143.9277207012611,-74.99286535541209]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[115.87353789573496,-14.400250740802694]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-29.13287445816456,-41.92410132711241]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[148.2151268801735,-11.956563276098233]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[68.52504223939638,-50.3069780026413]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[61.04974390294495,-82.99098862004999]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-1.9778454986500549,88.48795384648948]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[132.97569892947303,-18.197945509660872]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-46.434664245979064,-15.847299414146576]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[103.85577650831793,84.8245241543884]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[116.20985512973343,-72.58014253360143]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-84.14276121592428,39.32380023455736]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[167.99202997676994,59.84912933482924]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[145.21349874362696,-40.874452110286356]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[40.78190439421242,59.71659524084014]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[30.818833041139968,13.964520484213043]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-101.07837111795388,11.776614407541892]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[85.33879050387708,-1.8211058040616912]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[18.17165714770571,-35.80322592430113]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[33.68705761512178,-66.1836795020436]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-142.18279802591445,17.499453449704838]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-32.36019010087769,-80.74951195126101]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-171.4736485084086,62.039876984575955]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-108.79130212514202,70.91344476162534]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-123.94462349259467,-42.5720750899923]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-101.79874455925017,-70.63064356787535]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[158.23500516397246,62.58162679439841]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-78.92846641683336,-52.102625323575886]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[118.47955156603868,-75.23113584238706]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-4.117009063675363,27.32775752663199]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[47.9940094310702,-41.70922455468379]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[118.02540243535519,-24.024958318806263]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[126.45844501376448,40.225222459273574]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[46.095575115016786,50.19537168658261]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[53.981098234018845,70.9686868398325]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-123.80753246546506,-6.180030501925784]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[148.39957482179497,53.47441413191356]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-97.15475509864068,56.14413264374321]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-37.35726797264029,9.390545853566593]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[68.14829398665975,-57.61672336409853]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[142.0248332385989,14.436214736158263]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[35.46433545228567,51.3783524344255]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[62.21424514117686,-18.469927262172657]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-0.44607446695336783,64.56007912655703]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-43.64465756636359,29.567300327955]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-56.56995044670175,-71.07605188923556]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[138.31584057228696,-46.17498399607524]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[94.49281262574911,67.23993592825606]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-52.81985048385685,-62.23679402908813]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[115.39813852834835,83.30726648972964]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-152.8384831495821,-37.13669271964497]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-56.34286619516644,11.789178663854244]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-119.6196586653654,-78.66617206597016]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-157.97961294357825,-62.27348566406973]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-157.77430833700691,52.6416110103885]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-172.81748413269491,68.10453094455592]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[126.78455470915458,22.76844843540077]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[124.33951234857119,25.20059521114056]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-98.32676740536456,41.057305940372025]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[42.73364571270524,11.711161971359827]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[175.0831437931326,76.37818698544501]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[67.99441166798255,-33.9051802719499]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-162.5348302038685,22.24778350733955]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-25.785684853505764,-61.17848706951603]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-121.20840636192904,18.37271559656458]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[106.4158259776312,-69.02389601711933]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-137.072691601085,62.06890988855201]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-110.87943212813305,51.96460519658666]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-83.29255766786375,44.96674486762268]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-36.39291512935186,54.967262351971776]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-45.88560248125893,-83.0121890028711]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-0.9171797016339855,-76.68760889647275]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-143.51223903797634,33.79915374769206]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-166.46674803690928,30.765883073197713]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[21.58784190684002,-32.82572235854208]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-168.0962775194461,-84.68668621082192]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-156.29639103604254,-64.78491342794013]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-140.07056872371336,57.057922813739836]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-91.61181657850521,-56.87002472613748]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[169.3175087636519,-36.23240363741816]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[132.6976525110528,-15.67582192352905]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-96.20356214970218,-39.558761691759905]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-114.18818991395611,-13.193270799318428]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-64.9458902492152,-82.54401227314146]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[57.20529353510536,48.06568765798601]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-67.25541524987368,-59.62604826942611]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-164.41394358140673,-5.557529682140943]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-134.74781550227172,-14.46770237633027]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[56.73107616191273,-9.601456555477492]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-38.71042396518284,-70.53752547110018]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[71.19178401633677,-18.62168028146013]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-158.33749960380828,-14.228131676786958]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-68.73192054034834,11.463111276894615]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[149.88862938983618,-17.43671643379377]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-90.38989895960307,74.0948158621063]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-138.6524478769278,36.860573922841404]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[66.8087207311587,87.38574539176672]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-137.16403611723635,13.609108892656238]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-58.0787576620562,-45.656819764871805]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[34.64242206988593,4.896536277596071]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-122.75036368668317,27.390032545048875]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[139.59642330517536,-0.8361790379002665]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-44.95609801343284,-14.800569805482432]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[143.44329037960532,-76.30836103218506]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-34.82526234436272,77.5084720410569]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-8.85571259385542,-72.41898772465183]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-128.94974028721794,-13.872442605655756]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-98.96435262782651,-5.96349189581066]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[56.812530017284466,24.63009212221634]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-98.10296661645991,48.01901646833673]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-20.108303764485804,62.634932588032136]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-79.5187023573032,-3.741962805168959]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-21.381355717078854,-4.138332019420288]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-13.944293740839555,6.73115419624958]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-84.91673177258176,-73.78190809763487]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[110.20899030257772,83.07473341639596]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-76.1785793478868,77.62729851780838]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[102.45533903428486,-71.23573276503141]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[28.55081781776038,-66.87509508750925]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-90.9815792183709,-3.1303441038974578]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-124.03640040409687,-76.3228516948498]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[135.76191167878417,-14.647316044218748]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-51.29929391116478,-50.50803351529897]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-90.83546036767322,-47.845260775469114]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-78.27199602638846,-5.727781874317599]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-166.42758415040586,-35.374427268232445]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-105.70579545865415,-56.7898677909703]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[132.92087049654754,5.986636310344262]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-34.12000776153776,-37.8233604374587]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[153.48890673558918,48.54533064152512]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-177.19998135403878,65.11757816917759]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[51.76219118995236,-23.399094657386744]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-86.9656825928752,-45.15747489373153]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-161.05375786604202,-40.024621160009765]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[88.78210364905463,47.257223307731984]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[127.01953337361151,53.39331484229019]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-66.19544841588251,-72.78177351284552]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-148.23572013057148,-13.699749802422009]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-163.4904480132895,55.07906903354924]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[117.4383153344809,66.87265836819664]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-37.28255880867977,-61.740619790863306]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-20.606865347900758,-63.978828492195234]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[92.08330747090992,13.358719746997663]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-168.88383881223126,-88.26534909263118]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[118.79593076611233,-39.26406005693226]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[123.2286620226257,19.176292821784706]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[65.2566383521202,74.40175001074735]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[152.53244895075255,-60.89526039657967]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[23.670101239859527,79.97892983735672]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[76.90639299655113,-18.718296979262547]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-16.434102588826303,81.14838590052905]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-0.48241263068246276,-19.326970183078863]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[166.0770146317537,55.32451687266356]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[44.091982381851906,-18.061776258463844]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-62.135596257653596,61.38205045323859]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[77.77879153256856,-73.54453039781185]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-179.68514620114928,-20.738701477154372]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-124.8782756574879,68.4644288972731]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[119.1531689755124,45.227836700436264]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[175.0354203047252,25.28946785982305]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[3.9529694477677335,-70.2000352219346]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-145.66825995185198,-75.437341698581]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[63.39186996531453,86.24836768300835]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-39.35718145874311,19.267230520949372]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-115.46133804046431,64.31981054816362]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-124.20448134042186,84.56429996714996]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-86.65030222790674,-50.65532926127952]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[32.04522606463687,-57.656216536231966]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-166.63394340202606,-70.94111888282814]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-98.87613049737638,9.838786516278248]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[52.18141369506736,25.860926930101865]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-154.50373314480697,-58.59919446754799]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-127.75923264803065,21.222036673064206]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[21.02745484242215,-22.7672272046351]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[23.83596566609336,-79.8307423855834]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[144.5092450546486,32.501813126125164]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-28.446129612203876,-64.64371308205543]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[21.062244731129056,-40.23420976674345]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[74.93257241469765,-57.38843964941098]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[99.39525741584345,38.34285095816016]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[92.77330519303351,-51.88348699891928]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[85.10706256213875,-46.8483055436172]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[159.19877991848628,-77.48884965982114]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-150.91781812128517,-3.370603463821049]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-46.745058370485246,-50.03355974154423]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-57.842880652407594,-35.1998704320702]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-16.32450622308192,5.050915969233172]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[43.41966083293833,-5.5532802597072894]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-80.50018962317496,-43.79677181273735]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[157.63058994709402,-6.197538422490263]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-76.19311201929628,-39.62173160052461]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[134.17872119741529,66.03395326380192]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-58.709206678845916,27.578884523070922]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-3.7508634188511625,-22.63733935839804]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-33.50923239632988,87.57087982124473]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[167.0351193407785,-51.517398015638534]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[167.34466052388254,19.664826809456066]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-4.983112427926919,81.53507880974507]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-47.440002564396536,28.355541723789134]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-20.23881070467624,72.69308377052049]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[20.872182131778345,60.65214335653139]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-43.42727173327397,-75.98522951576123]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-131.24317729474345,74.30480390961668]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-67.97393481847206,-21.452505558219297]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[86.05524635048974,70.76176405343678]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-25.02088090626102,-83.89841418635477]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-70.70619813802524,9.293012704830957]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[136.8523177657593,-81.98518604958798]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-24.80084106803038,-61.39408338182605]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[96.39112214093079,-47.31588761236522]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-130.91154721422672,-62.39317653590273]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[96.69302367994173,84.81831777961641]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-62.69612358252903,73.92857397355446]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[36.085888205793424,88.19623705907155]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[146.15127377988188,21.119939468880155]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[115.69975392312523,52.106169992814735]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[117.12437696260233,50.633450410025006]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[78.03690459318167,25.093349844352886]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[111.66464673310823,25.743172267839576]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-76.1891927672624,43.64405377080228]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[178.05803932849614,-33.89063230457434]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-164.8902911821836,-39.158297859756594]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[118.81368776718443,69.45362439941802]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-161.64981794699293,10.84561681269902]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-150.86667870027816,-11.053556720703126]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-168.52798067768953,78.16705064282851]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-53.733025992313316,-45.75501970531106]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[28.02401831597048,67.64488867326105]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[64.99519729845811,-75.41753008731199]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-35.553695004779385,42.882767382231435]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[71.50728071448944,-40.14390812073772]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[57.186042097753344,-13.431805149680667]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-26.48328802422858,14.919531910677719]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-159.310299534434,-70.88903541090275]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-14.185479208720189,-12.584302352842588]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[145.93319829360314,60.33926066436778]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[103.83101392287466,88.98616543892196]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[29.74941593376112,6.881730457804478]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-168.1167377245246,74.99489489610268]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-47.66582662622247,55.89162433799111]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-15.173926224708012,-28.551492225026053]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-168.17147162508664,27.752558552992333]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[142.27566533067449,-52.4649639343418]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-178.139713518183,64.22025779784673]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[147.867230403598,-26.635034867895563]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[52.492035809623744,31.427814848521265]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-136.25423999843343,-61.94779345138131]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-163.53981111895007,-17.527328381591552]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[161.38669761971116,-84.8810025370632]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-42.68065830325797,-56.95408455031704]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[103.24489935818563,40.515440240630426]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-27.349729533204197,22.82168180343074]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-111.56071247708405,-87.00655645164757]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-80.66439143293046,-23.986499095183966]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[137.77250201451844,85.984352761386]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-14.737840297859677,-32.969565788896695]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[151.00938945702399,-50.067153942171515]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[8.00183785897194,-8.527969682588733]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[153.51299743092267,76.39964347031342]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-114.26488561301768,-88.34900913017918]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[115.18740127813226,-15.270937590584328]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-25.573475175041224,-28.75082771616279]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-116.61789493992876,67.30200249261813]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-154.09548805572973,-56.95295578107297]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[156.3998145842549,69.9268656726872]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[146.01121559900076,74.71833800878828]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[67.07460737224706,-0.8524654319782643]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[127.8359718714772,-86.67097891551961]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-166.85447903176615,9.143210724767222]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[60.43906040002717,-5.654948199718972]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[143.4116745106947,-35.89697160878518]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[64.22727662727362,-20.64347114480715]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-139.65173032697842,-41.40025097387484]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-10.702796777583998,42.29239594379252]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-122.82732330349864,-23.534800761982382]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-39.73042125597875,-58.0640731477155]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[115.36784417628007,8.39933657274826]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[63.90978621062626,11.20958794819742]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[8.987015711686315,-54.069466451689934]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-125.93655505344546,-75.91981875821358]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[129.4229764206238,-27.206921323365375]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-8.503804758650748,-19.067131511424044]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[150.52938772153206,-59.268329671008516]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[25.03407890071026,-30.824226431023988]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-85.56823855549366,64.0772394614836]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-175.94666268473586,51.107605965213686]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-157.41984046487084,79.68176001239273]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-147.78001924320867,30.82369491290631]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-104.12500472474753,11.258176557259784]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[36.51719145605651,61.824593086401336]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[47.261249638477274,6.446056312542052]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-79.41658539757654,-10.451265014492911]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[33.9919001385241,-87.51217783649426]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[42.99297897033563,32.88062530203285]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-19.9208626588739,23.756271506416024]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-154.01039202419878,-15.062902847434207]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-173.8496094488171,-12.12879895759508]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[174.3348985282082,-47.1252609058531]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[4.7705240474354405,53.98553861443948]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-88.15961587566504,-32.979114641644564]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-167.8539234288539,-83.61367503052529]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-92.79343490716835,32.34397740832629]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[74.5645585131474,-71.71793452258792]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[97.19056954515618,43.440872370306764]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[156.28899737595395,33.325722913703714]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[101.60063961180037,-55.55446366887629]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-56.91447180793819,77.16231555705939]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[61.96791543601911,11.24639094749783]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[135.3008264790216,10.798721063883079]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-151.77073792571034,-63.70789841184217]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-81.20267776843937,-23.834747902889767]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[130.1355787251288,26.74536058118786]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[99.3911104953032,9.819930418657176]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[116.56711294685773,-10.974651172796076]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-175.22156385571552,13.834702727081108]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-45.494564569676726,-73.82342886810709]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[34.18927357125557,83.57476776610105]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[69.95565340969573,56.88058931114621]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-130.2681001597441,-56.868198496743254]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-19.77700034055853,54.59401650323272]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-43.22721590678954,-73.92986167086441]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[54.25670313591002,-30.002956260206368]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-83.13531666714981,84.27255893046387]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[179.5025810890281,-60.14846993238712]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[52.837133153568516,-43.63272131578357]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-169.13881773738828,-4.123344125778345]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-83.94583686238609,-63.04643666335695]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[129.7662481897737,22.32467789561589]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[147.84793853443688,-58.102282940130664]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[151.43353786950564,22.676691080070324]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-20.183377030938637,18.089703427319073]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[146.98938856612955,-7.095322204161509]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-113.36861963290012,-72.49527612887084]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-30.780328319446156,-48.78306735651193]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-74.98846568826968,-37.540321201426025]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[46.20105580154054,53.800846011748995]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[178.78839581123648,53.04601688924891]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[62.39151663311607,-35.97487796271727]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-53.08337418045913,-40.060651617153326]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[11.69818993346909,76.93655062571057]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-44.93549736690859,-71.06575887658323]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[87.75611833783802,-18.98459490266084]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-139.77996367842272,75.57601492784354]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[115.74320470978319,-27.186540629281666]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[171.89721573863534,79.44611534678121]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[135.29800062452438,-6.433731106477456]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[6.189296968756484,34.396031694205014]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[161.37254224572905,55.19787775363667]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-5.555766770766581,89.15916605217534]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-131.07755362265152,12.292849359672505]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[98.96409104345851,67.89756168948749]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[135.91640460703212,40.089427258571845]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[107.13153008246832,13.779235149434426]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-106.08937373359707,-19.460890615104653]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-58.859380506614684,-22.1188189565882]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-93.33460622424882,45.51963219711006]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-114.40596369105216,34.012563978563165]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[148.57405756221215,-69.55793518358706]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-82.13199983163517,69.4047273345312]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-40.46531728025465,9.423524032233669]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-136.34196527305812,-53.88923485809161]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[137.63829818372068,-37.956141543374315]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[156.82332231453807,68.50649934616513]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-35.8812442925488,-81.87278761037827]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-70.11864538621155,18.425228263664213]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[37.02960372584969,-73.12697212621006]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-0.4071831122094238,-73.25590256184607]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-65.53721425994124,84.24221334333532]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-143.98747100062423,-53.70436434407298]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-123.9774189298057,-56.50270198204099]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[141.85022983683007,5.034233769119707]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[14.078847051024903,-81.5724128713891]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-13.356586928620393,-34.97647332801874]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-164.74251384181107,-82.65235000947011]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-137.98844797309212,51.02271520636166]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[15.905772708892671,21.314825509038393]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[38.5940075269537,72.3275447251368]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-90.68487488374609,58.14540009307414]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-124.81203506611482,-16.338459180691707]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[130.40173138500498,63.0900069687543]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-7.300951513393219,30.244525188544465]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[31.650464566531554,-15.2245114121295]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[94.25403712032374,47.50956953773842]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-54.818715338039574,62.413657501918955]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[107.85459863070406,-1.7766649877568241]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[129.54393320756074,-77.37871120462967]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[175.76752531483987,-50.363456012060716]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[167.69223124800516,-77.59864847306294]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[59.4004282669193,-34.11527488303557]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[20.755766772856408,8.45947916830086]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[138.16229776744734,-0.5589709325032155]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[31.273872347937484,-28.810406489882446]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-179.7473679147227,2.16591229868623]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[4.509801873683177,37.393227494286016]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[77.78472683636197,-26.029565067297252]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[122.25634139388688,-68.99938553127721]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-143.98699566007554,-45.08638358846308]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-81.4156275347008,2.517842779801769]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[153.9521953573221,-12.15315339017932]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[59.65031845208947,6.621093111393006]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[116.12608667948473,-59.41399973741132]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-92.9446201649005,-45.14249690768481]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[171.43937516653642,58.08371002497091]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-125.45596846220049,60.16965433777909]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[57.09390271231739,25.97828546306163]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[47.19785269078223,-6.865501270536489]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[116.60343735482405,51.828784574000174]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[59.262815662855644,-13.779193017456599]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-86.52490783300222,-41.01613810851694]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[24.79118961986139,-80.32044730814253]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[121.05724628942367,-89.68753642740928]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-22.923629124802467,-66.51854094262026]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[66.58881630705126,-10.462803913460155]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[135.35642233010242,68.81206025019159]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-82.09004692561169,-40.74110518497117]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[44.00130040406819,88.43666144248341]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-10.461726596277012,-73.72018063556185]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-135.1130201970025,-42.801462008540526]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[175.4516440687545,-24.586117320843556]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-83.93344844738537,-76.31959591451084]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[140.0902621921364,49.9804376808939]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[117.3529086958511,56.74678306483291]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-45.21576290776941,45.430383230164225]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-61.79269720562413,-45.0868033708835]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[57.69439149962195,-28.317609314435646]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-45.21759665332801,27.423928178951297]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[103.05494185666872,8.43191880169952]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[80.90658108096196,2.4221354086274305]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[42.89987937087474,-26.27746699525343]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[168.781166622296,-88.80906407040305]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[12.336420956850418,-82.53771868357376]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[153.93898920428586,66.5975809492675]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[120.1787953892819,49.15781860130664]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-155.20507869694327,-69.08237910172066]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[6.217512118446491,13.76456108431897]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-26.972187376382692,39.52551010167444]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-65.09641796506048,86.10069203842683]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-1.0386015358915035,3.961236525641323]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-26.981366729187837,41.63164590045403]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[158.40826692565224,-77.11931668411084]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[147.55325486715154,-0.7378753385852077]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-121.02557302461237,-12.852797361096933]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[143.6542497196445,33.977906948259744]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-108.04176538599559,-83.37501798993247]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[21.10146223557992,87.69500569926838]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[89.905920905405,55.829527207802336]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-19.249591361336105,51.5896823271716]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[26.169054319960026,17.40748848003459]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[51.09156188596657,-70.53943305295343]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-11.171302651956019,-28.30002584038434]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-11.152309945455139,-18.583003018226147]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[89.24832214799454,6.37025344402693]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-26.1870136136774,28.38702602039172]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[138.8322610357009,-12.299641895779088]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-33.78390651537123,89.26292825405558]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-133.5296036292762,-43.40426553384549]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-150.8461274284066,22.991401336368718]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-42.80565973874772,-72.23618361685782]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-3.2845678811452395,-47.73651092769137]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[101.43897932400512,55.3730513411963]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[69.17510253168005,14.062412498025134]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-154.62998526280555,-70.51865238422779]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[84.15324340942452,-50.272180140050125]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[175.38160083626374,34.921877705841524]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-12.81206555791222,-86.5515249384925]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[65.63040774503028,-3.5208431792261186]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[13.016196595776677,-21.47042713058287]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-95.84069990697459,1.950112838922231]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[156.84954664342587,46.12006205701243]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[4.315467282327763,74.36891560759757]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[106.62380056917722,-33.72546431545316]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-53.591361865257284,-12.082076957050628]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[107.13304914509904,31.63837311807095]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[90.67671608191124,-41.505400125170524]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-75.47831406186458,27.253875543321527]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[145.78110883187676,-49.75507442853016]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-149.62577276811155,-35.318075600980954]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-18.81965874117811,88.56831751882504]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[159.45687499894564,53.47238486224694]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[46.20693466250394,-3.447215945391049]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-62.50308612801867,38.65806953912612]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[75.11568458805154,-16.42416532287855]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[138.25174919450558,37.00247333068153]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-41.23438435912911,-14.357376895884792]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[151.91761045178484,-29.19316678155413]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-21.819285857316313,-17.334896809354788]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[118.41943196593543,60.86368699524066]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[82.92243232454831,-71.95715631953686]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[127.23826531944177,-18.727062092521468]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[129.32562134008057,-46.78556767438424]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-9.06695560436348,27.548902058447077]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[111.88034822323306,-39.890280523675784]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[109.65614048270155,-11.280107352414754]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[169.51695529110685,14.285638599502978]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[34.478571849171985,-6.285027613213883]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-68.3899085308515,89.62577343297369]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-47.07015327941321,-79.87601517000851]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-39.29771727161271,-12.445106159034758]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-41.38895661679378,47.96855703057005]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[49.24510097288061,34.600814640044675]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[54.72786597406631,31.6144623277006]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[30.121362716283215,78.3432878430897]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[1.7920152373395393,-28.40900891106147]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-41.31766158584013,-65.3655820474542]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[0.9335698783399948,24.268841839311094]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[130.93724714109078,-40.95797051298491]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-107.76241757110047,73.9001932726672]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-83.72768134332814,34.28496371115434]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-65.01225558925513,5.68408177883593]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[26.463072952199624,83.85516488663706]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[64.87975197328166,-37.69340355744355]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-51.937533727188246,2.1153954248867013]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[2.5940759508298328,77.09490399322718]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-20.819816446002104,50.17107064140063]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[129.15228775925408,74.89703810134833]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[159.73939789380753,35.74830286136273]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[34.938191033038535,10.177151636902519]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-72.52301124489891,3.9484751580107025]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[139.21528107551075,-25.14098385113355]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-143.67998660251692,10.598862496939475]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[75.23867244478555,48.32782729961331]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[8.853863234820869,69.70856592494447]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[31.204688816325685,80.95410041448318]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-147.79059850373514,9.380015555365816]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[179.01232028174633,82.18356071749854]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[80.44604434531367,-86.06656660536487]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-25.269120186093232,-15.50093630533441]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[26.040798447872547,-11.675343022156945]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[176.5155745210081,-37.530243015448065]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-154.5053014769047,-25.986485170792356]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-107.33116956874187,-48.051595878637514]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-79.59451775151855,67.27335630463547]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-80.07905069846126,24.627490274823604]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-40.815087585162885,64.74827063450172]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[177.15622596014282,55.94723077882618]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-154.32013926175705,6.054203508223366]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-152.52617440968544,30.75153043446005]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[82.32502029723413,51.7929163352901]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-167.97454191089346,7.29203141580161]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[68.50916276727092,62.69944431205739]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[162.64518938576705,-51.405520442579984]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-74.0025091259996,70.27753727482136]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[120.94340062033322,-87.72850769500474]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[173.85358638062735,-14.192993216177232]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-71.73931239228071,-62.07256073866491]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[94.21182616742648,53.914335903438946]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[152.446498648837,35.60215269822097]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[55.767587506335616,81.65772656384483]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[139.48412520009865,35.98116105613302]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-129.8329731597118,47.196521615521654]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[64.65597017802287,7.333242975078336]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-157.2423698471234,85.58380488151303]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[166.45854052236,83.28125899945871]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-165.85892030649478,-61.16125701586527]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-35.3482415465247,-3.7105634747617744]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[53.54001310043585,-56.137885296644974]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-73.19238282581637,73.70966951871402]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-155.42419781286048,85.76140168129244]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-93.0068278257192,-71.56204746079676]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-167.05741097461683,-29.08992421719146]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[83.00646798964486,4.04225437821963]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-96.40755319096593,49.053450408227526]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-112.83517664357956,78.63447238204128]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-122.28324320169732,-49.711482560875986]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-141.42768672681922,-74.87370978386066]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[61.453856533404476,-12.534202775063479]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-158.8139480529745,83.76595801751812]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-177.33937843615743,-26.884946458227734]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-149.21367772163543,-84.34282923519929]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-14.708444260542226,26.007231822208425]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[84.65328140934685,-89.04556059960703]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-126.70057600269313,67.19993550968718]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-63.578539217272876,-39.20353665419923]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[116.97649047977254,11.612893121647572]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-168.39597859890094,-7.176520027890239]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-83.54801174160602,75.60041831453867]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[169.08222229562273,61.08649000674105]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[89.53851277381662,85.50368569210505]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-47.43587083854676,69.80975249130913]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[64.67986390982989,-32.38277888377682]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-121.5022301249773,69.62473026330674]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[84.30365079881349,-38.78336771215365]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[112.92293665350968,53.44435208851027]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-125.56020224927215,42.52271902987475]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-39.51722259665548,26.301322752565316]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[13.727215708996265,0.5357052360126957]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-170.2859375027375,14.854268239823503]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[40.44432415400049,66.16771124042077]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[163.88591961775077,-80.21901038227104]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-18.86955299058272,25.828854385401247]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[125.80620546142583,36.54163896941496]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[160.14159424041355,-9.323182796993347]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[112.81132188754562,13.147281169683534]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-93.89355486641475,-81.80214698273579]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[148.53210373695072,49.94005276567554]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-114.40720077320896,67.32506015766414]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-0.3084944982630766,-32.40711568475365]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-138.97558310777006,-14.86327567250855]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[84.91139009819878,-4.506547073872946]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[178.28609981992517,-78.11795155127554]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-43.89662254225931,-21.54985281466525]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[62.359285877403806,35.78728794151067]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-15.66064958630534,3.8042148749330673]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-7.197995669785806,62.34552711482479]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-117.47759818598018,42.2311080931693]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[115.68617500729427,-34.79108058584426]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-23.56086932228271,-8.522658597759722]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-111.21815417055568,10.221443711788844]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-93.34291152658173,29.92729639384868]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[85.59049864101469,13.59518451491672]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-104.55713786930949,56.64974489662819]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[68.36055284233555,-40.26542577493315]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-138.18522318346547,63.532229776096734]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[125.7182744620651,42.83791957779601]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-94.03973523555923,84.21989238270243]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-126.95582191753856,10.450151578892314]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[53.33617930920302,64.51673858291662]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[56.022585217349004,-71.91443544647505]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-83.44794360942971,56.14268102736851]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-157.23690289334579,22.176402252210416]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[60.075265430815044,-7.810887540371323]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-80.88574243008308,56.264374176239244]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[156.44687777722632,-55.70497382795989]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[131.86353244570287,43.69613152319151]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[20.63545390409832,-62.172684282832186]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-171.51652607128236,86.11034329326047]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[13.634052951638385,-31.581613651170017]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[95.52818722244427,46.332635526254556]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[163.9835627993928,60.77542062662876]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-132.87663402057763,-24.662296743435]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-113.2183891036596,64.51733331142299]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-113.49873193302545,73.47491268712433]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-90.60244683385218,-15.37371951802903]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[59.019240710257804,-20.17418945944968]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-1.0023861011157287,55.85556133826032]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-98.49592759826821,66.48331739289246]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[90.99838924062979,-21.463776889272875]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[58.314272436836944,18.57893919073339]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-61.73670949243038,57.0026733970825]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-16.226551580990467,33.00844528046076]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-48.904329432978656,-3.858743127965778]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-170.53175372898116,-87.44873646312823]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-119.6897283246319,60.15975925322656]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-0.5957291836227441,67.32397111823147]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[69.76383028912413,39.194709421514666]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[66.03382556829322,-73.93539059727944]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-73.039428792872,39.655899929105594]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[69.53819002838726,19.028997739700774]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[25.074136731389565,-68.36731743047906]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-166.28380422825086,58.66211630181971]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-148.01910794006625,18.899281851777268]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[37.554835344834885,-8.082730514259433]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-23.344852701843948,-71.4006398783036]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-72.38452899882085,59.657081548057086]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[104.01972708970676,37.560961047780495]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[144.9128268769714,-76.13065644330601]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-102.64647540923146,-46.49666063283221]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-110.31567460753183,67.74935793409648]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-2.863068334971892,-7.097255786298953]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-179.96661050590296,-59.021293731433744]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-10.674662749539152,22.453089131355828]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[9.616057497180703,86.7208146106872]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[124.37101284966053,-4.310173515414655]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-104.96258145996795,-52.29229388568312]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-154.84315108308925,-70.09759671545815]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-63.141244270636896,-66.8482567598521]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[129.03724152441634,29.40298299271444]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-104.80479473681638,-5.626932922338508]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[103.22966537311446,-70.97076459088464]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-174.97801122738656,76.49653575009143]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[111.62139955966903,-33.7344978645058]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[158.69018997355357,59.099374071875914]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-58.40265735779247,4.156952542336456]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[178.10619635369792,82.28876829607164]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[102.81330988406194,58.644181294876205]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[130.0224540853144,-14.617215345807022]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-63.6773200781486,-5.3072492637039925]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[20.59561624028823,55.97740653542849]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[145.34653114998423,-0.30737383227630755]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[83.68720169308932,88.5430074666872]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[82.72430365181854,80.61239095750634]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-13.794912389278648,38.71606066584634]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[122.76063341212898,22.642816935337507]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-94.54678219512576,-6.506676109814551]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[105.9383084380006,56.93008860567418]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[78.99842143163801,-84.40016053098603]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-8.626251402987544,-63.027913307771804]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[118.0216404761902,30.261720013715735]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-38.40193273471364,43.43860109514538]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-14.338846048765213,-7.417043063002193]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-154.76427265106835,25.84563197920422]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[19.07690997509075,15.196618170015768]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[178.54044494787442,73.86068721693846]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[169.16528905553415,-16.11242716181036]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-11.312918141270298,74.15554691363518]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[49.39956267800543,-87.74972777310182]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[42.15613926234293,86.75045865554983]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-151.48015957723996,-74.03903299744961]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[164.70333917588604,-52.00852987564231]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[91.44938026715742,9.399908304825736]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-119.57642264422313,23.051162384667684]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[126.79605872857957,39.26251462537294]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[42.78676778239011,56.28222889497223]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[52.0677675005313,87.17255598592716]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[80.92365807310905,51.148299884969404]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-168.57600088233866,-33.20230227344964]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-70.05337330120213,-55.11340458752419]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[23.316075361272574,74.86292756163681]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-101.31092686458108,-43.262108269960365]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[138.77254158665298,-42.48085024432051]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[132.15679352755197,-2.4732567116271698]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[25.42739302384618,-57.36739218031762]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[170.91933946497196,24.6554544247484]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[163.67983890631965,-68.63866670317213]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-17.45300102674272,60.115936964604046]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-163.6720986453852,-59.27544683328588]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[103.12269310133763,-81.94786021251902]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[65.0197687178627,79.81613154673573]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-20.50923516256261,61.0683274999155]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[29.258789111526433,49.34750711611772]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-15.715893063384279,86.97991970568852]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[126.53302724925702,50.42239447822943]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[104.14798003358592,54.52477635604386]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-170.56596259156476,15.753501133493106]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-137.28234283142308,34.2866182224103]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[39.98668091752789,50.99986319688766]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-5.019243120642765,9.222426506209374]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-94.67185860833578,28.229951618970585]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[156.11351176715308,50.30338173887652]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-61.58191020384117,81.90591728179764]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-173.4801874261659,8.365155058423547]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[169.53369718209822,39.599786150003524]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[96.55413940856127,-63.66705143920976]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-151.60841157214685,-31.598323688959486]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-108.31187181268417,-23.728081608240696]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-57.89858315688551,79.06263768652147]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-27.10087726180256,-21.451835824050693]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-32.06632829029982,37.43680872938227]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-111.34952865916482,-30.700954501355437]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[126.83427193028075,45.81473300042905]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[176.22992561288586,-72.37784072109636]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[148.14711790276576,1.7565131552691904]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[41.416695881587096,23.57549595564568]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[58.44089629730409,-2.273430570888726]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[89.70536110033746,56.676301213439785]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-42.662441604801714,-29.895474849408437]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[163.55152498188755,66.08697983567461]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[16.389654629493666,-8.777085944850182]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[178.2780658836024,7.997038876882145]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-97.60866270287637,83.79504798052159]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-145.0152039493514,-57.47124718468864]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-80.87124362927283,-28.736860229560712]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-39.981503314845455,57.9368453809007]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-26.853160362660653,14.357267639136303]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-141.14668375618254,-84.1432243159947]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[98.9551807012928,-54.799847936234634]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[148.27861320521677,49.79021321185857]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[97.12763429086456,-8.949921236613307]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-22.361822263745303,-37.82877544971817]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[38.7155470076124,-43.13136775021742]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[123.7156272966424,-82.8609547468967]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-116.04820032044262,13.505774360817293]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[148.66988226216054,24.75217260123419]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[2.338448245599487,50.58480755204393]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[58.8625546473467,-54.062725837132454]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[42.315088125633906,83.6603599365099]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-145.7059862268191,-67.29501898041853]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[120.16231701170751,-39.20581186936942]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-109.65609949791214,-4.927260643202844]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[121.73296212728445,31.017945018342342]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-63.56529991480727,67.44393388806951]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[134.886674456276,-55.81541369121507]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[135.1935936888619,83.57044780257422]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-45.29897893317823,-35.88137130200411]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-151.12140526165217,57.394713990483716]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[179.8795464797712,-55.53728079803047]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-164.2850115158494,-57.07425180011188]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-92.6834059659534,31.75241393341896]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[163.0063982213062,50.46174139928017]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-1.5109509076525995,73.47862928116226]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-90.19768254451388,-81.87587079258049]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[164.03698341395886,-57.20360429921787]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-117.8908984472222,-36.85661418419499]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-76.63396536874355,75.46654510588866]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-137.11384948333585,-20.944615825519296]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[27.549348186753733,-41.24143752483883]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[5.8882136997245205,24.953092209827055]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-60.92822514570827,-53.42819241437253]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-110.7982641748912,73.11362498647041]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[18.005147765068948,6.606578723760106]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[102.49230488456102,-78.45677404334249]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-39.226266719603586,63.03621795553107]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-54.79105453322205,-63.885055017710165]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-60.22142199764942,-67.31455606760606]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-155.6745790461234,-31.276651705628723]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-48.59793131171237,56.08790348041517]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-27.47509023380121,32.31360316767686]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-138.0236065538707,-24.27004493709665]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-165.51181013860224,-14.298658058844481]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[43.14790236928267,64.04270081425511]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-61.82577313293886,3.5836037452005254]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-153.46214826613283,-24.877428653056075]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-160.5451327518163,-67.38126519691181]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[141.066804363372,-56.56154562126949]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[118.23143538537077,-72.20839506648583]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[82.59969813675907,-13.596872593089646]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[147.18530753126993,14.645377617181584]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[168.27392061356073,14.413954458139543]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-50.10665782150922,-77.50228247474821]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[28.367808597723805,6.3803111460193]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-89.59829671024326,41.014847862882206]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[58.35239377762565,-47.22554281417948]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[13.02210815664206,-8.315089199639193]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[164.2639748503703,-11.026519627431814]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-102.35418168954745,-5.706749663669073]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[106.63190788140622,67.04162954057414]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[85.01901598619851,15.897133687618282]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-139.0132666306482,52.93831416031908]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[67.45697629308425,88.93674240696821]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[13.744619383421304,-36.53887009926943]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-76.17840325555409,-21.201544961353406]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[86.54538277123774,82.24562985835223]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-68.39246646278768,-25.903082145588442]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[12.87475839087267,49.527570535152485]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[43.120815194138345,24.486207462595885]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[61.416346254507815,-44.7915754948088]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[178.46539373606672,75.37250520763354]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[91.00318194952186,21.53122274175137]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[30.612579199269412,-2.6320309944395115]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-173.59205563723145,-79.97846324879397]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-138.2900580274565,-18.968400523878785]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-37.742194948671056,-84.43184072134851]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[27.267377352005777,-76.5754252243422]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-12.916618962002255,24.231371928311496]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-74.82609477207231,-72.19601272254809]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[120.7342325215492,-46.27398602210843]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[102.00578805969309,-35.81073618564951]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[20.006963772801136,47.986844179422064]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[73.49917859397502,-70.95167213305776]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-109.38007100393308,43.82904606976852]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[39.87951550689655,-26.062135101054537]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[121.20181837427579,49.44652943853141]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[154.34477766499114,68.9060953056141]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-168.55666568442297,47.323633424543644]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[132.0530929208182,-83.51366560843975]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[101.38909345496268,15.808925798301466]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-26.002484207937506,-82.23208791904923]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[78.6230056845393,76.45219046038228]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[54.20958960140088,-25.182880857139573]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-128.1303142377929,-28.072661982614058]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[128.36304138250432,68.69030827733394]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[85.29744492582431,-33.618227327091724]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-30.798775048887556,-11.76032065453079]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[57.29935223034105,-56.81547597844948]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[91.27695803637324,14.742525411779459]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[7.786766854725169,56.014113029973196]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-52.587687550026814,29.705641787023843]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[128.3933617769031,29.815968757129035]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-106.00908541211587,-26.470189302850464]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-71.88774490048453,-71.32905619291715]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-164.80353233000804,-28.882106578139958]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-106.45698704072632,-80.66798464319183]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-138.82387983142996,61.33748055604995]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-130.18554594675896,-42.22763191289295]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-89.39626141515885,-50.08767345598521]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[67.84048105007903,47.9159538794716]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[39.26328866872443,15.348732722246945]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[81.27182229630502,56.5522739289503]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-134.86669257918496,71.59931841045831]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[43.769331899091895,44.356350794775324]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-157.69748839486297,-43.96458268888483]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[131.30973432068134,8.027102528239327]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-53.587829780341934,-46.04318786070643]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[141.33080785911238,4.6437269595731]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[120.43494920611172,10.907875920292689]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[20.341865845826,-29.960396351918437]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-55.07165130549812,-50.987415154683966]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[146.77414789129688,-20.345869978868777]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[92.41181199873782,39.885844989557945]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-4.201239859841053,-66.54998529710487]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[33.34100804140063,31.119090776413255]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[98.32846827065734,66.97524650396868]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[31.810679096065062,-28.399655666902888]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[80.1011242288579,-5.3547065014145945]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-166.45738566441514,-63.86358264657354]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[87.74920065571928,-7.778386057377902]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-118.21727311948922,15.032124793540618]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[135.927201795543,-15.299660865288441]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-104.29528368138395,-40.206933291517714]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-68.67651407425808,63.765460080930865]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[177.663921309484,57.71622343501538]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[40.21320923663567,-49.46777775499058]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-120.89966757055606,63.05993104773783]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-92.48905550371306,82.19509526217894]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[93.08501393240553,-42.5710245710858]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[100.62160248001895,71.78310985117636]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-155.6812824424169,52.31683529083127]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[30.99300247351156,55.5814127477169]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-147.92707536305326,48.61319545598781]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-70.5560609658576,78.25143968259434]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-111.0908070135768,-86.65217713640816]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-54.563494539109485,-7.360772078094113]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[65.88700572811605,42.20752866943668]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-25.852868682217736,-26.413052271661932]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[167.07972428105833,-72.31989292702535]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-108.0047094146438,19.549906856215415]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[164.75137853235765,44.908243073950686]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-91.40344072779283,65.72357009058949]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-93.90997958704534,58.234087891618394]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-64.9371924997813,50.779308425143036]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[168.50304067200437,75.51761262982004]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-103.71380569599431,58.86807328014232]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-91.34900720913797,61.1071651047452]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-51.05997527571599,-29.527497979303664]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[119.99949276523051,-47.59983845158048]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[99.35692633919773,-81.18744215480862]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-42.69816383831726,-77.06184989112117]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-137.93461860048095,67.45991308643059]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-151.79157959592965,49.532869997786285]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-33.36746431897324,-14.513480804756842]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-153.1408283062065,-79.82347451071911]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-67.57152603672137,12.949615595103833]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-30.94553784253364,57.71257986803953]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-77.68389917135725,-14.435156591827308]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[36.64012207358843,18.083333213062303]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-143.96651306470014,-7.367492587287381]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[138.98746023533286,-22.96147338791365]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-56.567963428730714,-30.113560449860355]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[114.22003959283668,-32.06587662731653]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-69.35236339394652,65.19170705580078]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[124.64904607393896,-52.95270273656519]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[128.22114440060014,-51.83124623710565]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-11.357519946542762,21.42932384115759]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-150.93219326501574,-52.72249282794564]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-112.36713112401782,-64.83143480873791]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-51.85764522887784,63.85158705497487]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-32.222003061834926,-20.553964730066987]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[167.72621097631378,29.233722953757365]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[56.06228711856849,-71.61212511658961]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[175.3673853690056,8.29526625865455]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-6.945545967810576,54.04252556256363]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-128.6482468604,63.41650193279859]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-178.93887545757755,-7.661447084878508]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[103.92816361249527,-70.43453722591691]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-166.2446511560406,-47.98429716244925]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[163.68006103185976,-31.86613197111744]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-164.21926920388276,46.052597465322314]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-166.7826849932883,-55.21419580663639]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[111.12833827535961,37.55651149712769]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[40.70065072892103,60.89352599411691]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-109.49496160146403,19.852927296488033]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[10.10644172713433,41.99332153424316]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-145.96198898315123,9.298664670086092]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[16.40336550567519,33.08147898499884]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[17.15393508258167,-43.955904354014706]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[124.935368343997,10.534474419830406]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[29.7272770765108,29.669602524901535]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-80.29480320048856,-31.125238585168653]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[51.90875648256588,62.130908142182726]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[174.39995538803396,-11.739958192139198]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[19.43284547660669,-13.68313790498894]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-79.41739146076497,-71.35630430612159]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-173.24077531431732,89.7251676073139]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-81.48620769203934,-69.82546137299829]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[70.97661105743056,28.101504623889717]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[54.09709609491998,-39.15569640905214]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-162.19906345760677,82.99045869982879]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[15.092358304530196,39.702696606229885]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-88.27894651094745,1.0050445166453992]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-83.12644164166569,-77.19337800996776]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[19.448755738270116,85.30530598215691]},"properties":{}},{"type":"Feature","geometry":{"type":"Point","coordinates":[137.78802039872954,15.04683967948595]},"properties":{}}]} \ No newline at end of file diff --git a/geoapi/tests/services/test_tippecanoe_service.py b/geoapi/tests/services/test_tippecanoe_service.py index e83d3963..ff607cc9 100644 --- a/geoapi/tests/services/test_tippecanoe_service.py +++ b/geoapi/tests/services/test_tippecanoe_service.py @@ -1,3 +1,8 @@ +import os +import re +import shutil +import subprocess +import tempfile from unittest.mock import patch import pytest @@ -56,3 +61,46 @@ def test_geojson_to_pmtiles_nonzero_exit_raises_runtimeerror(): TippecanoeService.geojson_to_pmtiles( "/tmp/in.geojson", "/tmp/out.pmtiles", "layer" ) + + +def _point_counts_by_zoom(pmtiles_path): + """Decode a .pmtiles and return {zoom: number of point features}.""" + decoded = subprocess.run( + ["tippecanoe-decode", pmtiles_path], + capture_output=True, + text=True, + check=True, + ).stdout + counts = {} + zoom = None + for line in decoded.splitlines(): + m = re.search(r'"zoom": (\d+)', line) + if m: + zoom = int(m.group(1)) + if '"Point"' in line and zoom is not None: + counts[zoom] = counts.get(zoom, 0) + line.count('"Point"') + return counts + + +@pytest.mark.worker +def test_geojson_to_pmtiles_keeps_features_at_all_zooms( + points_1000_geojson_path_fixture, +): + # tippecanoe drops features at low zoom by default, which would leave a + # scattered point layer nearly empty at the world view. --drop-rate=1 keeps + # every feature at every zoom; assert almost all 1000 points survive at each + # zoom, including the outermost (z0). + out_dir = tempfile.mkdtemp(prefix="geoapi_pmtiles_zoom_test_") + try: + pmtiles_path = os.path.join(out_dir, "out.pmtiles") + TippecanoeService.geojson_to_pmtiles( + points_1000_geojson_path_fixture, pmtiles_path, "pts" + ) + counts = _point_counts_by_zoom(pmtiles_path) + + assert counts, "no point features decoded from the archive" + assert 0 in counts, "archive has no world-view (zoom 0) tile" + for zoom, n in sorted(counts.items()): + assert n > 900, f"zoom {zoom} kept only {n} points (features dropped)" + finally: + shutil.rmtree(out_dir, ignore_errors=True) From 76f646b70ed163dca4e58a5324549420a2151a0e Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Wed, 22 Jul 2026 09:47:16 -0500 Subject: [PATCH 09/23] Connect current system and path with vector feature --- geoapi/services/features.py | 3 ++- geoapi/tests/api_tests/test_feature_service.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/geoapi/services/features.py b/geoapi/services/features.py index 93fc1139..d375d2d5 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -220,13 +220,14 @@ def fromVectorFile( feat.project_id = projectId feat.the_geom = from_shape(bbox_geom, srid=4326) feat.properties = metadata or {} - # TODO original_path, original_system are ignored but should not be ignored after WG-600 fa = FeatureAsset( uuid=asset_uuid, asset_type="vector", original_system=original_system, original_path=original_path, + current_system=original_system, + current_path=original_path, display_path=original_path, path=get_asset_relative_path(asset_path), feature=feat, diff --git a/geoapi/tests/api_tests/test_feature_service.py b/geoapi/tests/api_tests/test_feature_service.py index 47fe9216..8b44d8c3 100644 --- a/geoapi/tests/api_tests/test_feature_service.py +++ b/geoapi/tests/api_tests/test_feature_service.py @@ -222,6 +222,8 @@ def test_create_feature_shpfile( assert os.path.isfile(get_asset_path(asset.path)) assert asset.original_system == "system" assert asset.original_path == "foo" + assert asset.current_system == "system" + assert asset.current_path == "foo" assert asset.display_path == "foo" From ea41afcf848ef571f55795946e7e124d0424b47d Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 24 Jul 2026 11:25:21 -0500 Subject: [PATCH 10/23] Drop gpkg until we have a plan with styles --- geoapi/services/vectors.py | 20 +++++++++++-------- .../tests/api_tests/test_vectors_service.py | 8 +++++--- geoapi/utils/features.py | 3 --- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/geoapi/services/vectors.py b/geoapi/services/vectors.py index 96fba5a1..4cdffd99 100644 --- a/geoapi/services/vectors.py +++ b/geoapi/services/vectors.py @@ -9,12 +9,18 @@ logger = logging.getLogger(__name__) # Vector file extensions that are ingested via convert_to_geojson() -> tippecanoe -> PMTiles +# +# Deferred, not yet supported: +# * gpkg (GeoPackage) — a multi-layer container (and can hold raster tiles). +# Our "1 file = 1 Feature" model would silently ingest only the first layer. +# Proper support means exploding its layers into N Features (+ handling any +# embedded rasters as internal tile layers) — follow-on work. +# * parquet/geoparquet — geopandas needs pyarrow (a heavyweight dep) and the +# format is uncommon for our users. Re-adding is a one-dep change (add +# pyarrow, restore the exts + a read_parquet branch in _read_direct). SUPPORTED_VECTOR_EXTENSIONS = { "geojson", "json", - "gpkg", - "parquet", - "geoparquet", "shp", "gpx", } @@ -66,7 +72,7 @@ def convert_to_geojson( elif ext == "gpx": gdf = VectorService._read_gpx(fileObj) else: - gdf = VectorService._read_direct(fileObj, ext) + gdf = VectorService._read_direct(fileObj) if gdf.crs is None: # Inputs without a declared CRS (e.g. GeoJSON per RFC 7946) are WGS84 @@ -114,14 +120,12 @@ def _read_gpx(fileObj: IO) -> gpd.GeoDataFrame: return gpd.read_file(tmp_path, layer="tracks") @staticmethod - def _read_direct(fileObj: IO, ext: str) -> gpd.GeoDataFrame: - """Read a self-contained vector file (geojson/json/gpkg/parquet).""" + def _read_direct(fileObj: IO) -> gpd.GeoDataFrame: + """Read a self-contained vector file (geojson/json).""" with tempfile.TemporaryDirectory() as tmpdirname: tmp_path = os.path.join(tmpdirname, os.path.basename(fileObj.filename)) with open(tmp_path, "wb") as tmp: tmp.write(fileObj.read()) - if ext in ("parquet", "geoparquet"): - return gpd.read_parquet(tmp_path) return gpd.read_file(tmp_path) @staticmethod diff --git a/geoapi/tests/api_tests/test_vectors_service.py b/geoapi/tests/api_tests/test_vectors_service.py index 4e663f33..9f1d88cf 100644 --- a/geoapi/tests/api_tests/test_vectors_service.py +++ b/geoapi/tests/api_tests/test_vectors_service.py @@ -13,9 +13,11 @@ def test_supported_vector_extensions(): - assert {"geojson", "shp", "gpx", "gpkg", "parquet", "geoparquet"}.issubset( - SUPPORTED_VECTOR_EXTENSIONS - ) + assert {"geojson", "shp", "gpx"}.issubset(SUPPORTED_VECTOR_EXTENSIONS) + # Deferred formats are intentionally unsupported: gpkg (multi-layer, needs + # explode-into-N-features), parquet/geoparquet (no pyarrow dependency). + for deferred in ("gpkg", "parquet", "geoparquet"): + assert deferred not in SUPPORTED_VECTOR_EXTENSIONS @pytest.mark.worker diff --git a/geoapi/utils/features.py b/geoapi/utils/features.py index 7f92ee4d..fa19b28f 100644 --- a/geoapi/utils/features.py +++ b/geoapi/utils/features.py @@ -16,8 +16,6 @@ SHAPEFILE_FILE_EXTENSIONS = ("shp",) -ADDITIONAL_VECTOR_FILE_EXTENSIONS = ("gpkg", "parquet", "geoparquet") - RAPP_QUESTIONNAIRE_FILE_EXTENSIONS = ("rq",) RAPP_QUESTIONNAIRE_ARCHIVE_EXTENSIONS = "rqa" @@ -34,7 +32,6 @@ + GPX_FILE_EXTENSIONS + GEOJSON_FILE_EXTENSIONS + SHAPEFILE_FILE_EXTENSIONS - + ADDITIONAL_VECTOR_FILE_EXTENSIONS + RAPP_QUESTIONNAIRE_FILE_EXTENSIONS ) From 1b4222f94e0a60c532e0e5ba50b9203c2fb9ca0b Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 24 Jul 2026 15:39:04 -0500 Subject: [PATCH 11/23] Drop test and update comment --- geoapi/services/vectors.py | 10 +++++----- geoapi/tests/api_tests/test_vectors_service.py | 8 -------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/geoapi/services/vectors.py b/geoapi/services/vectors.py index 4cdffd99..d8c8fc84 100644 --- a/geoapi/services/vectors.py +++ b/geoapi/services/vectors.py @@ -11,13 +11,13 @@ # Vector file extensions that are ingested via convert_to_geojson() -> tippecanoe -> PMTiles # # Deferred, not yet supported: -# * gpkg (GeoPackage) — a multi-layer container (and can hold raster tiles). +# * TODO gpkg (GeoPackage) — a multi-layer container (and can also hold rasters). # Our "1 file = 1 Feature" model would silently ingest only the first layer. # Proper support means exploding its layers into N Features (+ handling any -# embedded rasters as internal tile layers) — follow-on work. -# * parquet/geoparquet — geopandas needs pyarrow (a heavyweight dep) and the -# format is uncommon for our users. Re-adding is a one-dep change (add -# pyarrow, restore the exts + a read_parquet branch in _read_direct). +# embedded rasters as internal tile layers) — follow-on work. And would be +# best have styling approach determined before progressing. +# * TODO gdb (Geodatabase) - Same thought process as above. Tracked in jirra +# task https://tacc-main.atlassian.net/browse/WG-105 SUPPORTED_VECTOR_EXTENSIONS = { "geojson", "json", diff --git a/geoapi/tests/api_tests/test_vectors_service.py b/geoapi/tests/api_tests/test_vectors_service.py index 9f1d88cf..202dc855 100644 --- a/geoapi/tests/api_tests/test_vectors_service.py +++ b/geoapi/tests/api_tests/test_vectors_service.py @@ -12,14 +12,6 @@ import pytest -def test_supported_vector_extensions(): - assert {"geojson", "shp", "gpx"}.issubset(SUPPORTED_VECTOR_EXTENSIONS) - # Deferred formats are intentionally unsupported: gpkg (multi-layer, needs - # explode-into-N-features), parquet/geoparquet (no pyarrow dependency). - for deferred in ("gpkg", "parquet", "geoparquet"): - assert deferred not in SUPPORTED_VECTOR_EXTENSIONS - - @pytest.mark.worker def test_convert_to_geojson_shapefile( shapefile_fixture, shapefile_additional_files_fixture From 8a3f5f2e4738ee2c8a4641ac1f3b1fafc5cbb70c Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 28 Jul 2026 19:29:26 -0500 Subject: [PATCH 12/23] Create and provide nightly public pmtiles archive for published DS maps --- geoapi/app.py | 19 ++ geoapi/celery_app.py | 6 + geoapi/custom/designsafe/published_export.py | 260 +++++++++++++++ geoapi/custom/designsafe/published_maps.py | 195 +++++++++++ geoapi/misc/README.md | 40 +++ geoapi/misc/dry_run_published_ds_maps.py | 144 ++++++++ geoapi/routes/__init__.py | 2 + geoapi/routes/published_ds_maps.py | 33 ++ geoapi/services/tippecanoe.py | 88 +++++ geoapi/tasks/published_ds_maps.py | 309 ++++++++++++++++++ .../test_published_ds_maps_routes.py | 28 ++ .../designsafe/test_published_export.py | 165 ++++++++++ .../custom/designsafe/test_published_maps.py | 122 +++++++ .../tests/services/test_tippecanoe_service.py | 80 +++++ .../tasks_tests/test_published_ds_maps.py | 215 ++++++++++++ geoapi/utils/client_backend.py | 31 ++ 16 files changed, 1737 insertions(+) create mode 100644 geoapi/custom/designsafe/published_export.py create mode 100644 geoapi/custom/designsafe/published_maps.py create mode 100644 geoapi/misc/README.md create mode 100644 geoapi/misc/dry_run_published_ds_maps.py create mode 100644 geoapi/routes/published_ds_maps.py create mode 100644 geoapi/tasks/published_ds_maps.py create mode 100644 geoapi/tests/api_tests/test_published_ds_maps_routes.py create mode 100644 geoapi/tests/custom/designsafe/test_published_export.py create mode 100644 geoapi/tests/custom/designsafe/test_published_maps.py create mode 100644 geoapi/tests/tasks_tests/test_published_ds_maps.py diff --git a/geoapi/app.py b/geoapi/app.py index 5c55e56c..bc955eb8 100644 --- a/geoapi/app.py +++ b/geoapi/app.py @@ -30,6 +30,7 @@ from geoapi.models import User from geoapi.routes import api_router from geoapi.settings import settings +from geoapi.log import logger from geoapi.db import litestar_sqlalchemy_config, managed_litestar_db_session from geoapi.exceptions import ( InvalidGeoJSON, @@ -296,8 +297,26 @@ async def retrieve_session_user_handler( } +def _enqueue_published_ds_maps_cold_start(_app: "Litestar") -> None: + """On startup, enqueue an initial published-DS-maps build if none exists yet.""" + # tests exercise the full app lifespan via TestClient; don't auto-enqueue a + # real task (which would hit the live DesignSafe API) during the test run + if settings.APP_ENV == "testing": + return + from geoapi.tasks.published_ds_maps import enqueue_generation_if_missing + + try: + enqueue_generation_if_missing() + except Exception: + # never let a cold-start convenience break app startup + logger.exception( + "Cold-start enqueue of published-DS-maps generation failed; continuing." + ) + + app = Litestar( route_handlers=[api_router], + on_startup=[_enqueue_published_ds_maps_cold_start], middleware=[ logging_middleware_config.middleware, cookie_session_config.middleware, diff --git a/geoapi/celery_app.py b/geoapi/celery_app.py index 3348ab21..38a465ef 100644 --- a/geoapi/celery_app.py +++ b/geoapi/celery_app.py @@ -1,4 +1,5 @@ from celery import Celery +from celery.schedules import crontab from datetime import timedelta from geoapi.settings import settings @@ -26,6 +27,7 @@ "geoapi.tasks.projects", "geoapi.tasks.external_data", "geoapi.tasks.file_location_check", + "geoapi.tasks.published_ds_maps", ) # Define the queues @@ -58,4 +60,8 @@ "task": "geoapi.tasks.external_data.refresh_projects_watch_users", "schedule": timedelta(minutes=30), }, + "generate_published_ds_maps_pmtiles": { + "task": "geoapi.tasks.published_ds_maps.generate_published_ds_maps_pmtiles", + "schedule": crontab(hour=3, minute=0), + }, } diff --git a/geoapi/custom/designsafe/published_export.py b/geoapi/custom/designsafe/published_export.py new file mode 100644 index 00000000..3cf7a167 --- /dev/null +++ b/geoapi/custom/designsafe/published_export.py @@ -0,0 +1,260 @@ +import json +from typing import List, Optional, Tuple +from urllib.parse import quote + +from geoapi.log import logging +from geoapi.models import Feature, TileServer +from geoapi.custom.designsafe.published_maps import PublishedMap +from geoapi.utils.client_backend import ( + get_deployed_geoapi_url, + get_deployed_hazmapper_url, +) + +logger = logging.getLogger(__name__) + +# DesignSafe published-data browser URL for a PRJ-#### project. +DESIGNSAFE_PUBLISHED_BROWSER_URL = ( + "https://www.designsafe-ci.org/data/browser/public/" + "designsafe.storage.published/{designsafe_project_id}" +) + +# asset_type (FeatureAsset) -> feature_type carried in the archive. Types not +# listed here fall through to a geometry-based point/shape classification (this +# covers plain geometry features and PMTiles "vector" assets, whose own detail +# lives in a separate archive). +_ASSET_TYPE_TO_FEATURE_TYPE = { + "image": "image", + "video": "video", + "point_cloud": "point_cloud", + "streetview": "streetview", + "questionnaire": "questionnaire", +} + + +class PublishedMapsExportService: + """Turn selected published maps into GeoJSON features for tiling. + + Emits one GeoJSON feature per Hazmapper feature, plus one footprint polygon + per internal COG (rasters are never tiled into this archive -- only their + bounds, carrying a layer descriptor the consumer can use to load the real + COG on demand). Every feature carries a small, stable, snake_case property + set that WG-671 (Recon Portal) depends on. + """ + + @classmethod + def build_features( + cls, database_session, published_maps: List[PublishedMap] + ) -> Tuple[List[dict], dict]: + """Build the GeoJSON features and summary stats. + + :return: ``(features, stats)`` where ``features`` is a list of GeoJSON + Feature dicts and ``stats`` has ``feature_count`` (DB features), + ``cog_count`` (COG footprints), ``project_count`` and ``total``. + """ + geoapi_base = get_deployed_geoapi_url() + hazmapper_base = get_deployed_hazmapper_url() + + features: List[dict] = [] + feature_count = 0 + cog_count = 0 + + for published in published_maps: + project = published.project + common = { + "project_uuid": str(project.uuid), + "project_name": project.name, + "ds_project_id": published.designsafe_project_id, + "ds_doi": published.designsafe_doi, + "ds_project_url": DESIGNSAFE_PUBLISHED_BROWSER_URL.format( + designsafe_project_id=published.designsafe_project_id + ), + } + + # regular Hazmapper features + for feature in ( + database_session.query(Feature) + .filter(Feature.project_id == project.id) + .all() + ): + try: + geometry = feature.geometry + except Exception: + logger.warning( + "Feature %s has unreadable geometry; skipping.", feature.id + ) + continue + if not geometry: + continue + properties = dict(common) + properties.update( + { + "feature_id": feature.id, + "feature_type": cls._feature_type(feature), + "hazmapper_url": cls._hazmapper_url( + hazmapper_base, project.uuid, feature.id + ), + "has_assets": bool(feature.assets), + "created_date": ( + feature.created_date.isoformat() + if feature.created_date + else None + ), + } + ) + thumbnail_url = cls._thumbnail_url(feature, geoapi_base) + if thumbnail_url: + properties["thumbnail_url"] = thumbnail_url + + features.append( + {"type": "Feature", "geometry": geometry, "properties": properties} + ) + feature_count += 1 + + # internal COG footprints (never the pixels -- just the outline + a + # descriptor the consumer uses to instantiate the real tile layer) + for tile_server in ( + database_session.query(TileServer) + .filter(TileServer.project_id == project.id) + .filter(TileServer.internal.is_(True)) + .filter(TileServer.kind == "cog") + .all() + ): + footprint = cls._cog_footprint( + tile_server, common, hazmapper_base, geoapi_base, project.uuid + ) + if footprint: + features.append(footprint) + cog_count += 1 + + stats = { + "feature_count": feature_count, + "cog_count": cog_count, + "project_count": len(published_maps), + "total": len(features), + } + logger.info( + "Public export built %s feature(s): %s DB features + %s COG footprints " + "across %s project(s).", + stats["total"], + feature_count, + cog_count, + stats["project_count"], + ) + return features, stats + + @staticmethod + def _feature_type(feature) -> str: + for asset in feature.assets: + mapped = _ASSET_TYPE_TO_FEATURE_TYPE.get(asset.asset_type) + if mapped: + return mapped + geometry_type = ((feature.geometry or {}).get("type") or "").lower() + return "point" if "point" in geometry_type else "shape" + + @staticmethod + def _thumbnail_url(feature, geoapi_base) -> Optional[str]: + """URL of an image feature's thumbnail, if it has one. + + Image assets are stored as ``.jpeg`` alongside a + ``.thumb.jpeg`` thumbnail (see FeaturesService.fromImage), and are + served at ``{geoapi}/assets/{asset.path}``. + """ + for asset in feature.assets: + if asset.asset_type == "image" and asset.path and asset.path.endswith( + ".jpeg" + ): + thumb_path = asset.path[: -len(".jpeg")] + ".thumb.jpeg" + return f"{geoapi_base}/assets/{thumb_path}" + return None + + @staticmethod + def _hazmapper_url(hazmapper_base, project_uuid, feature_id=None) -> str: + url = f"{hazmapper_base}/project-public/{project_uuid}" + if feature_id is not None: + url += f"?selectedFeature={feature_id}" + return url + + @classmethod + def _cog_footprint( + cls, tile_server, common, hazmapper_base, geoapi_base, project_uuid + ) -> Optional[dict]: + tile_options = tile_server.tileOptions or {} + bounds = tile_options.get("bounds") + # bounds is leaflet-style [[minLat, minLng], [maxLat, maxLng]] + if not bounds or len(bounds) != 2: + logger.warning( + "Internal COG tile_server %s has no usable bounds; skipping its " + "footprint.", + tile_server.id, + ) + return None + (south, west), (north, east) = bounds + ring = [ + [west, south], + [east, south], + [east, north], + [west, north], + [west, south], + ] + geometry = {"type": "Polygon", "coordinates": [ring]} + + ui_options = tile_server.uiOptions or {} + render_options = ui_options.get("renderOptions") or {} + + # Field-for-field mirror of hazmapper's TileServerLayer, JSON-encoded so + # the consumer can reconstruct the layer with no translation shim. (Vector + # tile attributes must be scalars, so this rides as a string; the flat + # fields below are convenience duplicates for quick styling/labeling.) + descriptor = { + "id": tile_server.id, + "name": tile_server.name, + "type": tile_server.type, + "kind": tile_server.kind, + "internal": tile_server.internal, + "uuid": str(tile_server.uuid) if tile_server.uuid else None, + "url": tile_server.url, + "attribution": tile_server.attribution, + "tileOptions": tile_options, + "uiOptions": ui_options, + } + + properties = dict(common) + properties.update( + { + "feature_id": None, + "feature_type": "cog", + "hazmapper_url": cls._hazmapper_url(hazmapper_base, project_uuid), + "has_assets": True, + "tile_server_id": tile_server.id, + "layer_name": tile_server.name, + "layer_type": tile_server.type, + "min_zoom": tile_options.get("minZoom"), + "max_zoom": tile_options.get("maxZoom"), + "max_native_zoom": tile_options.get("maxNativeZoom"), + "attribution": tile_server.attribution, + "opacity": ui_options.get("opacity"), + "colormap_name": render_options.get("colormap_name"), + "cog_url": cls._build_cog_url(tile_server, geoapi_base), + "tile_layer": json.dumps(descriptor), + } + ) + return {"type": "Feature", "geometry": geometry, "properties": properties} + + @staticmethod + def _build_cog_url(tile_server, geoapi_base) -> str: + """Pre-resolved TiTiler tile URL template for an internal COG. + + Mirrors the hazmapper frontend's resolveTileUrl: wrap the internal file + path as ``file://...`` (URL-encoded) and append any render options. The + ``{z}/{x}/{y}`` placeholders are left literal for the consumer. + """ + file_url = "file://" + tile_server.url + encoded = quote(file_url, safe="") + url = ( + f"{geoapi_base}/tiles/cog/tiles/WebMercatorQuad/" + f"{{z}}/{{x}}/{{y}}.png?url={encoded}" + ) + render_options = (tile_server.uiOptions or {}).get("renderOptions") or {} + for key, value in render_options.items(): + url += f"&{key}={quote(str(value), safe='')}" + return url diff --git a/geoapi/custom/designsafe/published_maps.py b/geoapi/custom/designsafe/published_maps.py new file mode 100644 index 00000000..03fd2b17 --- /dev/null +++ b/geoapi/custom/designsafe/published_maps.py @@ -0,0 +1,195 @@ +import time +from dataclasses import dataclass +from typing import Iterator, List, Optional + +import requests + +from geoapi.log import logging +from geoapi.models import Project +from geoapi.settings import settings + +logger = logging.getLogger(__name__) + +# DesignSafe publications API (public, unauthenticated). This is intentionally a +# module-level constant rather than a per-environment setting: "published" is a +# production DesignSafe curation act, and the maps embedded in a publication are +# tagged with the deployment that registered them (see below), so pointing at a +# single publications host and filtering by deployment is what keeps each geoapi +# tier scoped to its own maps. Overridable in tests. +DESIGNSAFE_PUBLICATIONS_API = "https://www.designsafe-ci.org/api/publications/v2" + +# Page size when walking the publications list. +PUBLICATIONS_PAGE_LIMIT = 100 + +# HTTP timeout for a single DesignSafe request. +REQUEST_TIMEOUT_SECONDS = 30 + + +@dataclass +class PublishedMap: + """A Hazmapper map that belongs to a published DesignSafe project *and* to + this geoapi deployment. + + ``project`` is the local geoapi Project (already confirmed public); the + remaining fields carry the DesignSafe publication context needed to enrich + exported features. + """ + + project: Project + map_uuid: str + map_name: Optional[str] + map_path: Optional[str] + designsafe_project_id: str # PRJ-#### + designsafe_project_title: Optional[str] + designsafe_doi: Optional[str] + + +class PublishedMapsService: + """Select the Hazmapper maps that back public, published DesignSafe projects + for the current deployment. + + The traversal mirrors the qgis plugin's discovery script + (``hazmapper-qgis-plugin/scripts/designsafe_hazmapper_discovery.py``): start + from the DesignSafe publications API, and for each publication read the + ``baseProject.hazmapperMaps`` it embeds. Each embedded map is tagged with the + ``deployment`` (the ``APP_ENV`` of the geoapi that registered it -- see + ``custom/designsafe/project.py``), which is how a single publication can + reference maps on more than one tier. + + A map is selected only when all three conditions hold: + 1. published -- it appears in a DesignSafe publication + 2. this tier -- ``deployment == settings.APP_ENV`` + 3. public -- the local Project joined by ``uuid`` has ``public = True`` + + A map that matches this deployment but whose local project is missing or not + public is logged and skipped, not raised -- one bad map must not sink the + nightly run. + """ + + # HTTP entry points are isolated here so tests can patch them without a + # network round-trip. + publications_api = DESIGNSAFE_PUBLICATIONS_API + + # Politeness delay between per-project detail requests. Set to 0 in tests. + inter_request_sleep_seconds = 0.1 + + @classmethod + def _get_json(cls, url: str) -> dict: + response = requests.get(url, timeout=REQUEST_TIMEOUT_SECONDS) + response.raise_for_status() + return response.json() + + @classmethod + def _iter_published_projects(cls) -> Iterator[dict]: + """Yield every published project (paged) from the publications list.""" + offset = 0 + while True: + data = cls._get_json( + f"{cls.publications_api}?offset={offset}&limit={PUBLICATIONS_PAGE_LIMIT}" + ) + projects = data.get("result", []) + if not projects: + return + for project in projects: + yield project + # a short page means we've reached the end + if len(projects) < PUBLICATIONS_PAGE_LIMIT: + return + offset += PUBLICATIONS_PAGE_LIMIT + + @classmethod + def _fetch_base_project(cls, designsafe_project_id: str) -> dict: + """Return the ``baseProject`` block for a single publication. + + Carries both the embedded ``hazmapperMaps`` and the publication ``dois``. + """ + detail = cls._get_json(f"{cls.publications_api}/{designsafe_project_id}") + return detail.get("baseProject") or {} + + @classmethod + def get_published_maps( + cls, database_session + ) -> List[PublishedMap]: + """Resolve the published, this-deployment, public maps to local Projects. + + :param database_session: SQLAlchemy session bound to this deployment's DB + :return: selected maps (may be empty -- e.g. a tier with no published maps) + """ + deployment = settings.APP_ENV + selected: List[PublishedMap] = [] + seen_uuids: set[str] = set() + deployment_matched = 0 + + for project in cls._iter_published_projects(): + designsafe_project_id = project.get("projectId") + if not designsafe_project_id: + continue + designsafe_title = project.get("title") + + base_project = cls._fetch_base_project(designsafe_project_id) + dois = base_project.get("dois") or [] + designsafe_doi = dois[0] if dois else None + + for hazmapper_map in base_project.get("hazmapperMaps") or []: + map_uuid = hazmapper_map.get("uuid") + if not map_uuid: + continue + + # Condition 2: only maps registered by *this* geoapi deployment. A map + # tagged for another tier lives in another tier's DB; skip quietly. + if hazmapper_map.get("deployment") != deployment: + continue + + deployment_matched += 1 + if map_uuid in seen_uuids: + continue + + # Condition 3: the map must resolve to a local, public project. + local_project = ( + database_session.query(Project) + .filter(Project.uuid == map_uuid) + .one_or_none() + ) + if local_project is None: + logger.warning( + "Published map tagged for this deployment label is not in " + "this database; skipping. This is expected when the label is " + "shared across instances (e.g. another 'local'/'dev'). " + f"deployment:{deployment} designsafe_project:{designsafe_project_id} " + f"map_uuid:{map_uuid}" + ) + continue + if not local_project.public: + # Published on DesignSafe but toggled non-public in Hazmapper. + # Log at error level (an inconsistency worth noticing) but do + # not raise -- just exclude this one map. + logger.error( + "Published DesignSafe map resolves to a non-public project; " + "excluding it from the public archive. " + f"deployment:{deployment} designsafe_project:{designsafe_project_id} " + f"map_uuid:{map_uuid} project_id:{local_project.id}" + ) + continue + + seen_uuids.add(map_uuid) + selected.append( + PublishedMap( + project=local_project, + map_uuid=map_uuid, + map_name=hazmapper_map.get("name"), + map_path=hazmapper_map.get("path"), + designsafe_project_id=designsafe_project_id, + designsafe_project_title=designsafe_title, + designsafe_doi=designsafe_doi, + ) + ) + + if cls.inter_request_sleep_seconds: + time.sleep(cls.inter_request_sleep_seconds) + + logger.info( + f"Selected {len(selected)} published, public map(s) for deployment " + f"'{deployment}' (out of {deployment_matched} deployment-matched " + "published map(s))." + ) + return selected diff --git a/geoapi/misc/README.md b/geoapi/misc/README.md new file mode 100644 index 00000000..04137aaf --- /dev/null +++ b/geoapi/misc/README.md @@ -0,0 +1,40 @@ +# `misc/` — one-off operational scripts + +Ad-hoc scripts run by hand (not part of the API or the Celery schedule). Today +there is just one. + +## `dry_run_published_ds_maps.py` + +A **non-destructive** dry run of the WG-703 published-DesignSafe-maps archive. It +runs the real selector → exporter → tiler against whatever database the +environment points at and prints: + +- project / feature / COG-footprint counts and a per-`feature_type` breakdown, +- per-feature property-payload sizes (which keys are heaviest — informs whether to + split project-level fields into a companion `projects.json`), +- the archive bounds, and +- the **total archive size**. + +It writes the archive to `/assets/tmp/published_ds_maps.pmtiles` (a scratch path, +never `/assets/public/`); delete it when done. + +### Run against the local dev stack + +```bash +docker exec -it geoapi_workers python misc/dry_run_published_ds_maps.py +``` + +### Run against production + +The deployed geoapi containers get their DB password, `APP_ENV`, etc. from a +secrets file on the host — see `Core-Portal-Deployments/geoapi-services/camino/docker-compose.yml`, +where `backend` and `celerybeat` use `env_file: /opt/portal/conf/secrets.env`. +Reuse that same file for a one-off container (org/tag come from `prod.env`): + +```bash +docker run --rm \ + --env-file /opt/portal/conf/secrets.env \ + -v /assets:/assets \ + taccwma/geoapi-workers: \ + python misc/dry_run_published_ds_maps.py +``` diff --git a/geoapi/misc/dry_run_published_ds_maps.py b/geoapi/misc/dry_run_published_ds_maps.py new file mode 100644 index 00000000..3586be1a --- /dev/null +++ b/geoapi/misc/dry_run_published_ds_maps.py @@ -0,0 +1,144 @@ +""" +Non-destructive dry run of the WG-703 published DesignSafe maps archive. + +Runs the REAL selector + exporter + tiler against whatever database the +environment is configured for, tiles into a scratch directory, and prints +counts, a per-layer breakdown, per-feature property-size stats, bounds, and the +total archive size. Use it to answer "how big is this archive and how many +projects/features?" before deploying -- e.g. pointed at production data. + +It is safe to run against production: + * DB access is READ-ONLY (SELECT via the ORM; nothing is written to the DB). + * Output goes ONLY to a scratch dir -- it never touches the real + assets/public area or manifest.json. + * It does NOT take the generation lock and does NOT enqueue anything. + +Requirements: this must run from a geoapi-workers image (it has tippecanoe + this +code) with the environment pointed at the target DB. APP_ENV must match the tier +whose maps you want (production maps are tagged deployment=production), because +the selector filters on `deployment == APP_ENV` and the URLs baked into the +archive come from APP_ENV. + +See misc/README.md for how to run it (local + production) and what it reports. +""" + +import argparse +import json +import os +import shutil +import tempfile +from collections import Counter, defaultdict + +from geoapi.custom.designsafe.published_export import PublishedMapsExportService +from geoapi.custom.designsafe.published_maps import PublishedMapsService +from geoapi.db import create_task_session +from geoapi.services.tippecanoe import ( + PUBLISHED_DS_MAPS_MAX_ZOOM, + PUBLISHED_DS_MAPS_MIN_ZOOM, + TippecanoeService, +) +from geoapi.settings import settings +from geoapi.tasks.published_ds_maps import _bounds, _write_layer_files +from geoapi.utils.assets import get_temp_dir +from geoapi.utils.client_backend import ( + get_deployed_geoapi_url, + get_deployed_hazmapper_url, +) + + +def _human_bytes(n): + for unit in ("B", "KB", "MB", "GB"): + if n < 1024 or unit == "GB": + return f"{n:.1f} {unit}" + n /= 1024 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--out", + default=None, + help="output .pmtiles path (default: " + "/assets/tmp/published_ds_maps.pmtiles; never the real assets/public area)", + ) + args = parser.parse_args() + + print("=" * 72) + print("WG-703 published DesignSafe maps archive -- DRY RUN (non-destructive)") + print("=" * 72) + print(f"APP_ENV : {settings.APP_ENV}") + print(f"DB host / name : {settings.DB_HOST} / {settings.DB_NAME}") + print(f"geoapi base URL : {get_deployed_geoapi_url()}") + print(f"hazmapper base URL : {get_deployed_hazmapper_url()}") + print(f"zoom range : z{PUBLISHED_DS_MAPS_MIN_ZOOM}-z{PUBLISHED_DS_MAPS_MAX_ZOOM}") + print("-" * 72) + + with create_task_session() as session: + published = PublishedMapsService.get_published_maps(session) + print(f"Published, public maps for this deployment: {len(published)}") + if not published: + print( + "\nNothing to measure. If you expected results, check that APP_ENV " + "matches the maps' deployment tag and that the DB is the right tier." + ) + return + features, stats = PublishedMapsExportService.build_features(session, published) + + print(f"DB features : {stats['feature_count']}") + print(f"COG footprints : {stats['cog_count']}") + print(f"Total features : {stats['total']}") + + # per-layer (feature_type) breakdown + by_type = Counter(f["properties"].get("feature_type") for f in features) + print("\nPer-layer (feature_type) counts:") + for feature_type, count in by_type.most_common(): + print(f" {feature_type:<14} {count}") + + # per-feature property payload analysis (informs the projects.json decision) + total_props_bytes = 0 + per_key_bytes = defaultdict(int) + for f in features: + props = f["properties"] + total_props_bytes += len(json.dumps(props)) + for key, value in props.items(): + per_key_bytes[key] += len(json.dumps({key: value})) + avg_props = total_props_bytes / len(features) if features else 0 + print( + f"\nProperty payload : {_human_bytes(total_props_bytes)} total, " + f"{avg_props:.0f} bytes/feature avg (pre-tiling, uncompressed)" + ) + print("Heaviest property keys (total bytes across all features):") + for key, nbytes in sorted(per_key_bytes.items(), key=lambda kv: -kv[1])[:8]: + print(f" {key:<16} {_human_bytes(nbytes)}") + + bounds = _bounds(features) + print(f"\nBounds [minx,miny,maxx,maxy]: {bounds}") + + # tile into a scratch path (NOT the real public area) + archive_path = args.out or os.path.join( + str(get_temp_dir()), "published_ds_maps.pmtiles" + ) + os.makedirs(os.path.dirname(archive_path), exist_ok=True) + work_dir = tempfile.mkdtemp(prefix="layers_", dir=os.path.dirname(archive_path)) + try: + layer_files = _write_layer_files(features, work_dir) + print(f"\nTiling {len(layer_files)} layer(s) into {archive_path} ...") + written = TippecanoeService.geojson_layers_to_pmtiles(layer_files, archive_path) + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + archive_size = os.path.getsize(archive_path) + print("-" * 72) + print(f"tippecanoe wrote : {written} features (source had {stats['total']})") + if written != stats["total"]: + print(" WARNING: count mismatch -- features were dropped!") + print(f"ARCHIVE SIZE : {_human_bytes(archive_size)} ({archive_size} bytes)") + print(f"Archive path : {archive_path}") + print( + f"\n(Delete {archive_path} when done. Nothing was written to the real " + "assets/public area or manifest.)" + ) + + +if __name__ == "__main__": + main() diff --git a/geoapi/routes/__init__.py b/geoapi/routes/__init__.py index 9ca26c91..70a6ca32 100644 --- a/geoapi/routes/__init__.py +++ b/geoapi/routes/__init__.py @@ -8,6 +8,7 @@ from .auth import AuthController from .webhooks import TaskStatusWebhookController from .websockets import websocket_handler +from .published_ds_maps import PublishedDsMapsController api_router = Router( path="/", @@ -20,5 +21,6 @@ AuthController, TaskStatusWebhookController, websocket_handler, + PublishedDsMapsController, ], ) diff --git a/geoapi/routes/published_ds_maps.py b/geoapi/routes/published_ds_maps.py new file mode 100644 index 00000000..581bf736 --- /dev/null +++ b/geoapi/routes/published_ds_maps.py @@ -0,0 +1,33 @@ +from litestar import Controller, get +from litestar.response import Response + +from geoapi.log import logging +from geoapi.tasks.published_ds_maps import read_manifest + +logger = logging.getLogger(__name__) + + +class PublishedDsMapsController(Controller): + """Unauthenticated discovery endpoint for the published DesignSafe maps archive. + + Cheap: it just serves the sidecar ``manifest.json`` written by the nightly + task -- it never streams the archive (nginx serves the .pmtiles directly) and + never builds it (a fresh environment 404s until the first run completes). + """ + + path = "/designsafe" + tags = ["public"] + + @get("/published-maps/pmtiles") + async def get_published_maps_pmtiles(self) -> Response: + """Return the manifest for the current combined public PMTiles archive of + published DesignSafe project maps (url + metadata), or 404 if none has + been generated yet. + """ + manifest = read_manifest() + if manifest is None: + return Response( + content={"detail": "No published-maps archive is available yet."}, + status_code=404, + ) + return Response(content=manifest, status_code=200) diff --git a/geoapi/services/tippecanoe.py b/geoapi/services/tippecanoe.py index a9f75a59..e03ecbb0 100644 --- a/geoapi/services/tippecanoe.py +++ b/geoapi/services/tippecanoe.py @@ -1,5 +1,7 @@ import os +import re import subprocess +from typing import List, Optional, Tuple from geoapi.log import logging @@ -8,6 +10,9 @@ # tippecanoe binary; overridable for environments where it is not on PATH TIPPECANOE_BIN = os.environ.get("TIPPECANOE_BIN", "tippecanoe") +PUBLISHED_DS_MAPS_MIN_ZOOM = 6 +PUBLISHED_DS_MAPS_MAX_ZOOM = 16 + # Floor for tippecanoe's guessed maximum zoom (-zg). # # `-zg` picks a max zoom from how far apart / how detailed the features are. @@ -91,3 +96,86 @@ def geojson_to_pmtiles( if result.stderr: logger.debug("tippecanoe output: %s", result.stderr) return output_path + + @staticmethod + def geojson_layers_to_pmtiles( + layers: List[Tuple[str, str]], + output_path: str, + min_zoom: int = PUBLISHED_DS_MAPS_MIN_ZOOM, + max_zoom: int = PUBLISHED_DS_MAPS_MAX_ZOOM, + ) -> Optional[int]: + """ + Tile per-layer GeoJSON files into one PMTiles archive with NO dropping. + + Used for the combined published DesignSafe maps archive (WG-703). Unlike + ``geojson_to_pmtiles`` (per-feature vector ingest, which guesses maxzoom + and may shed on tile overflow), this pins a moderate zoom range and + disables every form of feature thinning, so every marker is present at + every zoom. Deep display is handled by client overzoom -- see + ``PUBLISHED_DS_MAPS_MAX_ZOOM``. + + Each entry becomes its own vector-tile layer, so the consumer can style + and toggle by feature type. + + :param layers: list of ``(layer_name, geojson_path)`` + :param output_path: path where the .pmtiles archive will be written + :param min_zoom: minimum zoom (default z6) + :param max_zoom: maximum zoom (default z16) + :return: the feature count tippecanoe reports writing (for the caller's + source-vs-archive count check), or ``None`` if it can't be parsed + :raises RuntimeError: if the tippecanoe binary is missing or exits non-zero + :raises ValueError: if ``layers`` is empty + """ + if not layers: + raise ValueError("geojson_layers_to_pmtiles requires at least one layer") + + cmd = [ + TIPPECANOE_BIN, + "-o", + output_path, + "--force", # overwrite output_path if it already exists + "-Z", + str(min_zoom), + "-z", + str(max_zoom), + # --- no feature dropping, at any zoom (WG-703 requires every marker) --- + "--drop-rate=1", # keep every feature at every zoom (no rate thinning) + "--no-feature-limit", # don't cap features per tile + "--no-tile-size-limit", # don't drop to keep tiles under the size cap + "--no-tiny-polygon-reduction", # keep small footprints as-is + "--no-line-simplification", # don't move vertices (COG footprints stay put) + ] + for layer_name, geojson_path in layers: + # `-L name:file` reads file into its own named vector-tile layer + cmd += ["-L", f"{layer_name}:{geojson_path}"] + + logger.info("Running tippecanoe (public archive): %s", " ".join(cmd)) + try: + result = subprocess.run(cmd, capture_output=True, text=True) + except FileNotFoundError as e: + raise RuntimeError( + f"tippecanoe binary not found (looked for '{TIPPECANOE_BIN}'). " + "Install tippecanoe or set the TIPPECANOE_BIN environment variable." + ) from e + + if result.returncode != 0: + logger.error( + "tippecanoe failed (exit %s): %s", result.returncode, result.stderr + ) + raise RuntimeError( + f"tippecanoe failed with exit code {result.returncode}: " + f"{result.stderr.strip()}" + ) + + return TippecanoeService._parse_written_feature_count(result.stderr) + + @staticmethod + def _parse_written_feature_count(stderr: str) -> Optional[int]: + """Pull the feature count out of tippecanoe's summary line. + + tippecanoe prints e.g. ``"3 features, 81 bytes of geometry ..."`` to + stderr; this is the number of distinct input features it kept (it does + not match the earlier ``"Read N million features"`` progress line). + """ + match = re.search(r"(\d+) features,", stderr or "") + return int(match.group(1)) if match else None diff --git a/geoapi/tasks/published_ds_maps.py b/geoapi/tasks/published_ds_maps.py new file mode 100644 index 00000000..07552d7f --- /dev/null +++ b/geoapi/tasks/published_ds_maps.py @@ -0,0 +1,309 @@ +import json +import os +import shutil +import tempfile +import time +from datetime import datetime, timezone +from typing import List, Optional, Tuple + +import redis +import shapely.geometry + +from geoapi.celery_app import app +from geoapi.custom.designsafe.published_export import PublishedMapsExportService +from geoapi.custom.designsafe.published_maps import PublishedMapsService +from geoapi.db import create_task_session +from geoapi.log import logger +from geoapi.services.tippecanoe import ( + PUBLISHED_DS_MAPS_MAX_ZOOM, + PUBLISHED_DS_MAPS_MIN_ZOOM, + TippecanoeService, +) +from geoapi.settings import settings +from geoapi.utils.assets import get_temp_dir +from geoapi.utils.client_backend import get_deployed_geoapi_url + +# Layout of the shared public asset area (served unauthenticated by nginx). +PUBLIC_ASSET_SUBDIR = "public" +MANIFEST_FILENAME = "manifest.json" + +# Versioned, lexicographically-sortable archive name: +# published_ds_maps_20260724T030000Z.pmtiles +ARCHIVE_PREFIX = "published_ds_maps_" +ARCHIVE_SUFFIX = ".pmtiles" + +# Bumped when the manifest/property schema changes in a way consumers must notice. +SCHEMA_VERSION = 1 + +# Prune archives older than this, but never the one the manifest references. 26h +# (not 24h) leaves margin for a late run so a predecessor isn't removed early; in +# steady state this keeps two generations. +PRUNE_AGE_SECONDS = 26 * 3600 + +# Redis lock so a nightly beat, a startup enqueue, and a manual trigger can't run +# two tippecanoe builds at once. Auto-expires so a crashed run doesn't wedge it. +GENERATION_LOCK_KEY = "published_ds_maps:generation" +GENERATION_LOCK_TIMEOUT_SECONDS = 2 * 3600 + + +def public_asset_dir() -> str: + return os.path.join(settings.ASSETS_BASE_DIR, PUBLIC_ASSET_SUBDIR) + + +def manifest_path() -> str: + return os.path.join(public_asset_dir(), MANIFEST_FILENAME) + + +def read_manifest() -> Optional[dict]: + """Return the current manifest dict, or None if there isn't one yet.""" + try: + with open(manifest_path()) as f: + return json.load(f) + except (FileNotFoundError, ValueError): + return None + + +def _public_asset_url(filename: str) -> str: + return f"{get_deployed_geoapi_url()}/assets/{PUBLIC_ASSET_SUBDIR}/{filename}" + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _archive_filename(now: datetime) -> str: + # UTC, explicit Z, no colons/spaces -> filesystem- and URL-safe and sortable + return f"{ARCHIVE_PREFIX}{now.strftime('%Y%m%dT%H%M%SZ')}{ARCHIVE_SUFFIX}" + + +def _write_layer_files(features: List[dict], work_dir: str) -> List[Tuple[str, str]]: + """Write features grouped by ``feature_type`` to newline-delimited GeoJSON. + + :return: list of ``(layer_name, file_path)`` for tippecanoe named layers + """ + grouped: dict = {} + for feature in features: + layer = feature["properties"].get("feature_type") or "feature" + grouped.setdefault(layer, []).append(feature) + + layer_files: List[Tuple[str, str]] = [] + for layer, layer_features in grouped.items(): + path = os.path.join(work_dir, f"{layer}.geojson") + with open(path, "w") as f: + for feature in layer_features: + f.write(json.dumps(feature)) + f.write("\n") + layer_files.append((layer, path)) + return layer_files + + +def _bounds(features: List[dict]) -> Optional[List[float]]: + """[minx, miny, maxx, maxy] over all feature geometries (lon/lat).""" + minx = miny = maxx = maxy = None + for feature in features: + geometry = feature.get("geometry") + if not geometry: + continue + fminx, fminy, fmaxx, fmaxy = shapely.geometry.shape(geometry).bounds + minx = fminx if minx is None else min(minx, fminx) + miny = fminy if miny is None else min(miny, fminy) + maxx = fmaxx if maxx is None else max(maxx, fmaxx) + maxy = fmaxy if maxy is None else max(maxy, fmaxy) + if minx is None: + return None + return [minx, miny, maxx, maxy] + + +def _verify_feature_count(source_count: int, written_count: Optional[int]) -> None: + """Guard against silent dropping (the main failure mode of tiling). + + Raises if tippecanoe wrote fewer features than we handed it, so the caller + aborts before overwriting a good archive/manifest. + """ + if written_count is None: + logger.warning( + "Could not parse tippecanoe's feature count; skipping the " + "source-vs-archive count check (expected %s).", + source_count, + ) + return + if written_count != source_count: + raise RuntimeError( + "Public archive feature count mismatch: handed tippecanoe " + f"{source_count} feature(s) but it wrote {written_count}. Features " + "were dropped -- aborting without touching the existing archive." + ) + logger.info("Public archive feature count verified: %s.", source_count) + + +def _prune_old_archives(keep_filename: str) -> None: + """Delete archives older than PRUNE_AGE_SECONDS, never the referenced one.""" + directory = public_asset_dir() + manifest = read_manifest() or {} + referenced = os.path.basename(manifest.get("url", "")) if manifest else "" + now = time.time() + for name in os.listdir(directory): + if not (name.startswith(ARCHIVE_PREFIX) and name.endswith(ARCHIVE_SUFFIX)): + continue + if name == keep_filename or name == referenced: + continue + path = os.path.join(directory, name) + try: + age = now - os.path.getmtime(path) + except OSError: + continue + if age > PRUNE_AGE_SECONDS: + try: + os.remove(path) + logger.info( + "Pruned stale public archive %s (age %.1fh).", name, age / 3600 + ) + except OSError: + logger.exception("Failed to prune public archive %s.", name) + + +@app.task(queue="heavy") +def generate_published_ds_maps_pmtiles(): + """Nightly: build the combined public PMTiles archive for published DS maps. + + Selects this deployment's published, public maps, exports their features + (plus internal-COG footprints) to GeoJSON, tiles them with no dropping, and + atomically publishes the archive + a sidecar manifest into the shared public + asset area. Fails loudly and leaves the previous archive/manifest untouched + on any error. Guarded by a Redis lock so runs can't overlap. + + Runs nightly via celery beat; on a fresh environment app startup also enqueues + one build if no manifest exists yet (see on_startup in app.py). To regenerate + manually (the Redis lock keeps it from colliding with the nightly): + + # enqueue onto the heavy worker (returns immediately) + docker exec -it geoapi_workers python -c \ + "from geoapi.tasks.published_ds_maps import \ + generate_published_ds_maps_pmtiles as t; t.delay()" + + # or run synchronously (blocks until the archive is built) + docker exec -it geoapi_workers python -c \ + "from geoapi.tasks.published_ds_maps import \ + generate_published_ds_maps_pmtiles as t; t.apply()" + + For a non-destructive dry run that only measures counts/size, see + geoapi/misc/dry_run_published_ds_maps.py. + """ + start_time = time.time() + redis_client = redis.Redis( + host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=0 + ) + lock = redis_client.lock( + GENERATION_LOCK_KEY, timeout=GENERATION_LOCK_TIMEOUT_SECONDS + ) + if not lock.acquire(blocking=False): + logger.info( + "Public pmtiles generation is already running; skipping this trigger." + ) + return + + try: + with create_task_session() as session: + published_maps = PublishedMapsService.get_published_maps(session) + if not published_maps: + logger.info( + "No published maps for deployment '%s'; leaving any existing " + "archive and manifest untouched.", + settings.APP_ENV, + ) + return + features, stats = PublishedMapsExportService.build_features( + session, published_maps + ) + + if not features: + logger.info( + "Published maps yielded no features; leaving any existing archive " + "and manifest untouched." + ) + return + + os.makedirs(public_asset_dir(), exist_ok=True) + # Work under ASSETS_BASE_DIR/tmp so the finished archive is on the SAME + # filesystem as the public dir and os.replace() is atomic. + work_dir = tempfile.mkdtemp(prefix="published_ds_maps_", dir=str(get_temp_dir())) + now = _utc_now() + filename = _archive_filename(now) + try: + layer_files = _write_layer_files(features, work_dir) + staged_archive = os.path.join(work_dir, filename) + written_count = TippecanoeService.geojson_layers_to_pmtiles( + layer_files, staged_archive + ) + _verify_feature_count(len(features), written_count) + + final_archive = os.path.join(public_asset_dir(), filename) + # atomic within the same filesystem: a partial archive is never readable + os.replace(staged_archive, final_archive) + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + manifest = { + "url": _public_asset_url(filename), + "generated_at": now.isoformat(), + "feature_count": stats["total"], + "project_count": stats["project_count"], + "bounds": _bounds(features), + "zoom_min": PUBLISHED_DS_MAPS_MIN_ZOOM, + "zoom_max": PUBLISHED_DS_MAPS_MAX_ZOOM, + "schema_version": SCHEMA_VERSION, + } + _write_manifest(manifest) + + # prune only after the manifest points at the new archive + _prune_old_archives(keep_filename=filename) + + logger.info( + "Public pmtiles archive published: %s (%s features across %s projects) " + "in %.1fs.", + filename, + stats["total"], + stats["project_count"], + time.time() - start_time, + ) + except Exception: + logger.exception( + "Failed to generate public pmtiles archive; previous archive and " + "manifest left untouched." + ) + raise + finally: + try: + lock.release() + except Exception: # noqa: E722 + # lock may have already expired; nothing actionable + pass + + +def _write_manifest(manifest: dict) -> None: + """Atomically replace the sidecar manifest.json.""" + tmp_path = manifest_path() + ".tmp" + with open(tmp_path, "w") as f: + json.dump(manifest, f) + os.replace(tmp_path, manifest_path()) + + +def enqueue_generation_if_missing() -> bool: + """Cold-start self-heal: enqueue a build if no manifest exists yet. + + Called on app startup so a fresh environment produces an archive within one + task run instead of waiting for the nightly beat. Never runs generation + inline (that would block the worker for minutes); the Redis lock keeps this + from overlapping the nightly or a manual trigger. + + :return: True if a generation task was enqueued + """ + if read_manifest() is not None: + return False + try: + generate_published_ds_maps_pmtiles.delay() + logger.info("No public pmtiles manifest present; enqueued initial generation.") + return True + except Exception: + logger.exception("Failed to enqueue initial public pmtiles generation.") + return False diff --git a/geoapi/tests/api_tests/test_published_ds_maps_routes.py b/geoapi/tests/api_tests/test_published_ds_maps_routes.py new file mode 100644 index 00000000..678fdb59 --- /dev/null +++ b/geoapi/tests/api_tests/test_published_ds_maps_routes.py @@ -0,0 +1,28 @@ +from unittest.mock import patch + +ROUTE = "/designsafe/published-maps/pmtiles" + + +def test_get_manifest_returns_200_when_present(test_client): + manifest = { + "url": "https://example.org/assets/public/published_ds_maps_x.pmtiles", + "generated_at": "2026-07-24T03:00:00+00:00", + "feature_count": 5, + "project_count": 2, + "bounds": [-98.0, 30.0, -96.0, 31.0], + "zoom_min": 6, + "zoom_max": 16, + "schema_version": 1, + } + # unauthenticated (guest) access, no token + with patch("geoapi.routes.published_ds_maps.read_manifest", return_value=manifest): + resp = test_client.get(ROUTE) + assert resp.status_code == 200 + assert resp.json() == manifest + + +def test_get_manifest_returns_404_on_cold_start(test_client): + with patch("geoapi.routes.published_ds_maps.read_manifest", return_value=None): + resp = test_client.get(ROUTE) + assert resp.status_code == 404 + assert "detail" in resp.json() diff --git a/geoapi/tests/custom/designsafe/test_published_export.py b/geoapi/tests/custom/designsafe/test_published_export.py new file mode 100644 index 00000000..f7485ac1 --- /dev/null +++ b/geoapi/tests/custom/designsafe/test_published_export.py @@ -0,0 +1,165 @@ +import json +import uuid + +from geoalchemy2.shape import from_shape +from shapely.geometry import Point + +from geoapi.models import Feature, FeatureAsset, Project, TileServer +from geoapi.custom.designsafe.published_export import PublishedMapsExportService +from geoapi.custom.designsafe.published_maps import PublishedMap + + +def _project(db_session): + project = Project(name="pub", description="pub", tenant_id="test", public=True) + db_session.add(project) + db_session.commit() + return project + + +def _point_feature(db_session, project): + feature = Feature(project_id=project.id, properties={}) + feature.the_geom = from_shape(Point(-97.0, 30.0), srid=4326) + db_session.add(feature) + db_session.commit() + return feature + + +def _image_feature(db_session, project): + feature = Feature(project_id=project.id, properties={}) + feature.the_geom = from_shape(Point(-96.0, 31.0), srid=4326) + asset_uuid = uuid.uuid4() + asset = FeatureAsset( + uuid=asset_uuid, + asset_type="image", + path=f"{project.id}/{asset_uuid}.jpeg", + feature=feature, + ) + feature.assets.append(asset) + db_session.add(feature) + db_session.commit() + return feature + + +def _cog_tile_server(db_session, project): + ts = TileServer( + project_id=project.id, + name="my-cog", + type="xyz", + kind="cog", + internal=True, + uuid=uuid.uuid4(), + url=f"/assets/{project.id}/{uuid.uuid4()}.cog.tif", + attribution="TACC", + tileOptions={ + "minZoom": 0, + "maxZoom": 22, + "maxNativeZoom": 22, + "bounds": [[30.0, -97.0], [31.0, -96.0]], + "renderOptions": {"colormap_name": "terrain"}, + }, + uiOptions={"opacity": 1, "renderOptions": {"colormap_name": "terrain"}}, + ) + db_session.add(ts) + db_session.commit() + return ts + + +def _published(project, doi="10.17603/ds2-test"): + return PublishedMap( + project=project, + map_uuid=str(project.uuid), + map_name="My Map", + map_path="/", + designsafe_project_id="PRJ-1", + designsafe_project_title="Published One", + designsafe_doi=doi, + ) + + +def test_build_features_emits_features_cog_and_enriched_properties(db_session): + project = _project(db_session) + point = _point_feature(db_session, project) + image = _image_feature(db_session, project) + _cog_tile_server(db_session, project) + + features, stats = PublishedMapsExportService.build_features( + db_session, [_published(project)] + ) + + assert stats == { + "feature_count": 2, + "cog_count": 1, + "project_count": 1, + "total": 3, + } + assert len(features) == 3 + + by_type = {} + for f in features: + by_type.setdefault(f["properties"]["feature_type"], []).append(f) + + # every feature carries the shared publication context + for f in features: + props = f["properties"] + assert props["project_uuid"] == str(project.uuid) + assert props["ds_project_id"] == "PRJ-1" + assert "designsafe.storage.published/PRJ-1" in props["ds_project_url"] + assert props["ds_doi"] == "10.17603/ds2-test" + assert "/project-public/" in props["hazmapper_url"] + + # point feature: geometry-derived type, deep link, no thumbnail + pt = by_type["point"][0] + assert pt["properties"]["feature_id"] == point.id + assert pt["properties"]["has_assets"] is False + assert f"selectedFeature={point.id}" in pt["properties"]["hazmapper_url"] + assert "thumbnail_url" not in pt["properties"] + + # image feature: asset-derived type + thumbnail pointer + img = by_type["image"][0] + assert img["properties"]["feature_id"] == image.id + assert img["properties"]["has_assets"] is True + assert img["properties"]["thumbnail_url"].endswith(".thumb.jpeg") + assert "/assets/" in img["properties"]["thumbnail_url"] + + # cog footprint: polygon + resolvable tile url + field-for-field descriptor + cog = by_type["cog"][0] + assert cog["geometry"]["type"] == "Polygon" + assert cog["properties"]["feature_id"] is None + assert cog["properties"]["layer_name"] == "my-cog" + assert cog["properties"]["max_zoom"] == 22 + assert cog["properties"]["colormap_name"] == "terrain" + cog_url = cog["properties"]["cog_url"] + assert "/tiles/cog/tiles/WebMercatorQuad/{z}/{x}/{y}.png?url=" in cog_url + assert "file%3A%2F%2F" in cog_url # url-encoded file:// + assert "colormap_name=terrain" in cog_url + + descriptor = json.loads(cog["properties"]["tile_layer"]) + assert descriptor["internal"] is True + assert descriptor["kind"] == "cog" + assert descriptor["type"] == "xyz" + assert descriptor["tileOptions"]["maxZoom"] == 22 + + +def test_external_tile_layers_are_not_exported(db_session): + project = _project(db_session) + # an external (user-attached) layer -- must be excluded + external = TileServer( + project_id=project.id, + name="external-basemap", + type="tms", + kind=None, + internal=False, + url="https://example.com/{z}/{x}/{y}.png", + attribution="someone else", + tileOptions={"bounds": [[30.0, -97.0], [31.0, -96.0]]}, + uiOptions={}, + ) + db_session.add(external) + db_session.commit() + + features, stats = PublishedMapsExportService.build_features( + db_session, [_published(project)] + ) + + assert stats["cog_count"] == 0 + assert features == [] diff --git a/geoapi/tests/custom/designsafe/test_published_maps.py b/geoapi/tests/custom/designsafe/test_published_maps.py new file mode 100644 index 00000000..1a755297 --- /dev/null +++ b/geoapi/tests/custom/designsafe/test_published_maps.py @@ -0,0 +1,122 @@ +import logging +import uuid +from unittest.mock import patch + +from geoapi.models import Project +from geoapi.custom.designsafe.published_maps import PublishedMapsService +from geoapi.settings import settings + + +def _make_project(db_session, name, public): + """Create + commit a project and return it (uuid is auto-generated).""" + project = Project(name=name, description=name, tenant_id="test", public=public) + db_session.add(project) + db_session.commit() + return project + + +def _fake_designsafe_api(pages, details): + """Build a _get_json replacement that routes by URL. + + :param pages: dict of offset -> list result page + :param details: dict of PRJ id -> hazmapperMaps list + """ + + def _get_json(url): + if "/publications/v2?" in url or url.endswith("/publications/v2"): + # extract offset from the query string + offset = int(url.split("offset=")[1].split("&")[0]) + return {"result": pages.get(offset, [])} + designsafe_project_id = url.rsplit("/", 1)[1] + return {"baseProject": {"hazmapperMaps": details.get(designsafe_project_id, [])}} + + return _get_json + + +def test_get_published_maps_selects_only_matching_maps_and_logs_exclusions( + db_session, caplog +): + """A map is selected only if it passes all three conditions -- published, + deployment == APP_ENV, and the local project is public -- and the two + inconsistency paths (non-public, not-in-DB) are logged.""" + this_deployment = settings.APP_ENV + + public_project = _make_project(db_session, "public-map", public=True) + private_project = _make_project(db_session, "private-map", public=False) + missing_uuid = str(uuid.uuid4()) + + pages = { + 0: [ + {"projectId": "PRJ-1", "title": "Published One"}, + {"projectId": "PRJ-2", "title": "Published Two"}, + {"projectId": "PRJ-3", "title": "Published Three"}, + {"projectId": "PRJ-4", "title": "Published Four"}, + ] + } + details = { + # selected: published + this deployment + public + "PRJ-1": [ + { + "uuid": str(public_project.uuid), + "name": "Public Map", + "path": "/", + "deployment": this_deployment, + } + ], + # excluded: resolves to a non-public project -> error logged + "PRJ-2": [ + {"uuid": str(private_project.uuid), "deployment": this_deployment} + ], + # excluded: registered by a different deployment tier + "PRJ-3": [ + {"uuid": str(public_project.uuid), "deployment": "some-other-tier"} + ], + # excluded: this deployment, but no such project in the DB -> warning + "PRJ-4": [{"uuid": missing_uuid, "deployment": this_deployment}], + } + + with patch.object( + PublishedMapsService, "_get_json", side_effect=_fake_designsafe_api(pages, details) + ), patch.object(PublishedMapsService, "inter_request_sleep_seconds", 0), caplog.at_level( + logging.WARNING + ): + selected = PublishedMapsService.get_published_maps(db_session) + + # only the public, this-deployment map survives all three conditions + assert len(selected) == 1 + result = selected[0] + assert result.project.id == public_project.id + assert result.map_uuid == str(public_project.uuid) + assert result.map_name == "Public Map" + assert result.designsafe_project_id == "PRJ-1" + assert result.designsafe_project_title == "Published One" + + # the non-public inconsistency is logged at error level (not raised) + assert any( + record.levelno == logging.ERROR and "non-public" in record.getMessage() + for record in caplog.records + ) + # the missing map is logged at warning level + assert any( + record.levelno == logging.WARNING and missing_uuid in record.getMessage() + for record in caplog.records + ) + + +def test_selector_returns_empty_when_no_deployment_matches(db_session): + """A tier with no published maps of its own yields an empty selection.""" + public_project = _make_project(db_session, "public-map", public=True) + + pages = {0: [{"projectId": "PRJ-1", "title": "Published One"}]} + details = { + "PRJ-1": [ + {"uuid": str(public_project.uuid), "deployment": "a-different-deployment"} + ] + } + + with patch.object( + PublishedMapsService, "_get_json", side_effect=_fake_designsafe_api(pages, details) + ), patch.object(PublishedMapsService, "inter_request_sleep_seconds", 0): + selected = PublishedMapsService.get_published_maps(db_session) + + assert selected == [] diff --git a/geoapi/tests/services/test_tippecanoe_service.py b/geoapi/tests/services/test_tippecanoe_service.py index ff607cc9..4c09e585 100644 --- a/geoapi/tests/services/test_tippecanoe_service.py +++ b/geoapi/tests/services/test_tippecanoe_service.py @@ -82,6 +82,86 @@ def _point_counts_by_zoom(pmtiles_path): return counts +def test_geojson_layers_to_pmtiles_builds_no_drop_command(): + with patch("geoapi.services.tippecanoe.subprocess.run") as mock_run: + mock_run.return_value = _completed(stderr="42 features, 1234 bytes of geometry") + count = TippecanoeService.geojson_layers_to_pmtiles( + [("points", "/tmp/points.geojson"), ("cog", "/tmp/cog.geojson")], + "/tmp/out.pmtiles", + ) + + # the feature count is parsed from tippecanoe's summary line + assert count == 42 + cmd = mock_run.call_args[0][0] + assert cmd[0] == "tippecanoe" + assert "-o" in cmd and cmd[cmd.index("-o") + 1] == "/tmp/out.pmtiles" + # fixed moderate zoom range (overzoom handles deep display) + assert cmd[cmd.index("-Z") + 1] == "6" + assert cmd[cmd.index("-z") + 1] == "16" + # no dropping of any kind + assert "--drop-rate=1" in cmd + assert "--no-feature-limit" in cmd + assert "--no-tile-size-limit" in cmd + assert "--no-tiny-polygon-reduction" in cmd + assert "--no-line-simplification" in cmd + # crucially NOT the overflow-shedding used by per-feature ingest + assert "--drop-densest-as-needed" not in cmd + # one named layer per entry + assert "points:/tmp/points.geojson" in cmd + assert "cog:/tmp/cog.geojson" in cmd + + +def test_geojson_layers_to_pmtiles_requires_layers(): + with pytest.raises(ValueError, match="at least one layer"): + TippecanoeService.geojson_layers_to_pmtiles([], "/tmp/out.pmtiles") + + +def test_parse_written_feature_count_ignores_progress_line(): + stderr = "Read 0.00 million features\n7 features, 10 bytes of geometry\n" + assert TippecanoeService._parse_written_feature_count(stderr) == 7 + assert TippecanoeService._parse_written_feature_count("no summary here") is None + + +@pytest.mark.worker +def test_geojson_layers_to_pmtiles_no_drop_multilayer(): + # tile a point layer and a polygon (COG-footprint-like) layer into one + # archive; assert the reported count matches input and both layers survive. + out_dir = tempfile.mkdtemp(prefix="geoapi_published_ds_maps_test_") + try: + points_path = os.path.join(out_dir, "points.geojson") + shapes_path = os.path.join(out_dir, "shapes.geojson") + with open(points_path, "w") as f: + f.write( + '{"type":"Feature","geometry":{"type":"Point",' + '"coordinates":[-97.7,30.3]},"properties":{"feature_type":"point"}}\n' + ) + with open(shapes_path, "w") as f: + f.write( + '{"type":"Feature","geometry":{"type":"Polygon","coordinates":' + "[[[-97.8,30.2],[-97.6,30.2],[-97.6,30.4],[-97.8,30.4],[-97.8,30.2]]]}," + '"properties":{"feature_type":"cog"}}\n' + ) + + pmtiles_path = os.path.join(out_dir, "out.pmtiles") + count = TippecanoeService.geojson_layers_to_pmtiles( + [("points", points_path), ("cogs", shapes_path)], pmtiles_path + ) + + assert os.path.isfile(pmtiles_path) + assert count == 2 # both input features present, none dropped + + decoded = subprocess.run( + ["tippecanoe-decode", pmtiles_path], + capture_output=True, + text=True, + check=True, + ).stdout + assert '"points"' in decoded + assert '"cogs"' in decoded + finally: + shutil.rmtree(out_dir, ignore_errors=True) + + @pytest.mark.worker def test_geojson_to_pmtiles_keeps_features_at_all_zooms( points_1000_geojson_path_fixture, diff --git a/geoapi/tests/tasks_tests/test_published_ds_maps.py b/geoapi/tests/tasks_tests/test_published_ds_maps.py new file mode 100644 index 00000000..5a0cb11b --- /dev/null +++ b/geoapi/tests/tasks_tests/test_published_ds_maps.py @@ -0,0 +1,215 @@ +import json +import os +import time +from unittest.mock import MagicMock, patch + +import pytest + +from geoapi.settings import settings +from geoapi.tasks import published_ds_maps +from geoapi.tasks.published_ds_maps import ( + ARCHIVE_PREFIX, + ARCHIVE_SUFFIX, + generate_published_ds_maps_pmtiles, +) + +FEATURES = [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [-97.7, 30.3]}, + "properties": {"feature_type": "point", "feature_id": 1}, + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [[-97.8, 30.2], [-97.6, 30.2], [-97.6, 30.4], [-97.8, 30.4], [-97.8, 30.2]] + ], + }, + "properties": {"feature_type": "cog", "feature_id": None}, + }, +] +STATS = {"feature_count": 1, "cog_count": 1, "project_count": 1, "total": 2} + + +def _write_dummy_archive(layers, output_path, *args, **kwargs): + with open(output_path, "wb") as f: + f.write(b"PMTILES-DUMMY") + return len(FEATURES) # tippecanoe "wrote" 2 features + + +class _ContextManagerMock: + def __enter__(self): + return MagicMock() + + def __exit__(self, *exc): + return False + + +def _patchers( + isolated_assets_dir, + published_maps=("sentinel",), + build_return=(FEATURES, STATS), + tile_side_effect=_write_dummy_archive, + lock_acquired=True, +): + """Common patch set. Returns a list of started patchers (caller stops them).""" + lock = MagicMock() + lock.acquire.return_value = lock_acquired + redis_client = MagicMock() + redis_client.lock.return_value = lock + + patchers = [ + patch.object(settings, "ASSETS_BASE_DIR", isolated_assets_dir), + patch.object(published_ds_maps.redis, "Redis", return_value=redis_client), + patch.object( + published_ds_maps, "create_task_session", return_value=_ContextManagerMock() + ), + patch.object( + published_ds_maps.PublishedMapsService, + "get_published_maps", + return_value=list(published_maps), + ), + patch.object( + published_ds_maps.PublishedMapsExportService, + "build_features", + return_value=build_return, + ), + patch.object( + published_ds_maps.TippecanoeService, + "geojson_layers_to_pmtiles", + side_effect=tile_side_effect, + ), + ] + return patchers, redis_client + + +def _run(patchers): + for p in patchers: + p.start() + try: + return generate_published_ds_maps_pmtiles() + finally: + for p in reversed(patchers): + p.stop() + + +def test_generate_writes_archive_and_manifest(tmp_path): + assets_dir = str(tmp_path) + patchers, _ = _patchers(assets_dir) + _run(patchers) + + public_dir = os.path.join(assets_dir, "public") + archives = [ + n + for n in os.listdir(public_dir) + if n.startswith(ARCHIVE_PREFIX) and n.endswith(ARCHIVE_SUFFIX) + ] + assert len(archives) == 1 + filename = archives[0] + + with open(os.path.join(public_dir, "manifest.json")) as f: + manifest = json.load(f) + assert manifest["feature_count"] == 2 + assert manifest["project_count"] == 1 + assert manifest["zoom_min"] == 6 + assert manifest["zoom_max"] == 16 + assert manifest["schema_version"] == 1 + assert manifest["url"].endswith(f"/assets/public/{filename}") + assert len(manifest["bounds"]) == 4 + # bounds span both features + assert manifest["bounds"][0] == pytest.approx(-97.8) + assert manifest["bounds"][2] == pytest.approx(-97.6) + + +def test_generate_skips_when_no_published_maps(tmp_path): + assets_dir = str(tmp_path) + patchers, _ = _patchers(assets_dir, published_maps=()) + _run(patchers) + + # nothing written, no manifest + assert not os.path.exists(os.path.join(assets_dir, "public", "manifest.json")) + + +def test_generate_skips_when_lock_held(tmp_path): + assets_dir = str(tmp_path) + patchers, _ = _patchers(assets_dir, lock_acquired=False) + # build_features must not be called if the lock isn't acquired + with patch.object( + published_ds_maps.PublishedMapsExportService, "build_features" + ) as build: + for p in patchers: + p.start() + try: + generate_published_ds_maps_pmtiles() + finally: + for p in reversed(patchers): + p.stop() + build.assert_not_called() + + +def test_generate_aborts_on_count_mismatch_without_touching_manifest(tmp_path): + assets_dir = str(tmp_path) + + def _wrong_count(layers, output_path, *a, **k): + with open(output_path, "wb") as f: + f.write(b"PMTILES-DUMMY") + return 1 # fewer than the 2 we handed it -> silent drop + + patchers, _ = _patchers(assets_dir, tile_side_effect=_wrong_count) + with pytest.raises(RuntimeError, match="feature count mismatch"): + _run(patchers) + + # no archive placed, no manifest written + public_dir = os.path.join(assets_dir, "public") + assert not os.path.exists(os.path.join(public_dir, "manifest.json")) + placed = [ + n + for n in os.listdir(public_dir) + if n.startswith(ARCHIVE_PREFIX) and n.endswith(ARCHIVE_SUFFIX) + ] + assert placed == [] + + +def test_enqueue_generation_if_missing_enqueues_only_when_absent(): + with patch.object(published_ds_maps, "read_manifest", return_value={"url": "x"}), patch.object( + published_ds_maps.generate_published_ds_maps_pmtiles, "delay" + ) as delay: + assert published_ds_maps.enqueue_generation_if_missing() is False + delay.assert_not_called() + + with patch.object(published_ds_maps, "read_manifest", return_value=None), patch.object( + published_ds_maps.generate_published_ds_maps_pmtiles, "delay" + ) as delay: + assert published_ds_maps.enqueue_generation_if_missing() is True + delay.assert_called_once() + + +def test_generate_prunes_old_archives_but_keeps_recent(tmp_path): + assets_dir = str(tmp_path) + public_dir = os.path.join(assets_dir, "public") + os.makedirs(public_dir, exist_ok=True) + + old = os.path.join(public_dir, f"{ARCHIVE_PREFIX}20200101T000000Z{ARCHIVE_SUFFIX}") + recent = os.path.join(public_dir, f"{ARCHIVE_PREFIX}20990101T000000Z{ARCHIVE_SUFFIX}") + for path in (old, recent): + with open(path, "wb") as f: + f.write(b"old") + # make `old` older than the 26h window; leave `recent` fresh + stale = time.time() - (30 * 3600) + os.utime(old, (stale, stale)) + + patchers, _ = _patchers(assets_dir) + _run(patchers) + + remaining = set(os.listdir(public_dir)) + assert os.path.basename(old) not in remaining # pruned (stale) + assert os.path.basename(recent) in remaining # kept (fresh) + # the newly generated archive is present too + assert any( + n.startswith(ARCHIVE_PREFIX) + and n.endswith(ARCHIVE_SUFFIX) + and n not in {os.path.basename(old), os.path.basename(recent)} + for n in remaining + ) diff --git a/geoapi/utils/client_backend.py b/geoapi/utils/client_backend.py index 7c415a29..3f529415 100644 --- a/geoapi/utils/client_backend.py +++ b/geoapi/utils/client_backend.py @@ -80,3 +80,34 @@ def get_deployed_geoapi_url(): else: logger.exception(f"Unknown/unsupported APP_ENV:{settings.APP_ENV}") raise ApiException(f"Unknown APP_ENV:{settings.APP_ENV}") + + +def get_deployed_hazmapper_url(): + """ + Get the Hazmapper frontend base URL for the current environment. + + Mirrors the frontend's host + basePath convention (see hazmapper + getHazmapperBase/getBasePath): a single host, distinguished by a path + prefix per environment. Used to build public-map deep links, e.g. + ``{base}/project-public/{uuid}?selectedFeature={id}``. + """ + # basePath per environment mirrors the frontend's getBasePath. Note that + # production lives under /hazmapper (not root) -- this matches the canonical + # public-map links DesignSafe itself publishes + # (https://hazmapper.tacc.utexas.edu/hazmapper/project-public//). + hazmapper_urls = { + "local": "http://localhost:4200", + "production": "https://hazmapper.tacc.utexas.edu/hazmapper", + "staging": "https://hazmapper.tacc.utexas.edu/staging", + "dev": "https://hazmapper.tacc.utexas.edu/dev", + "testing": "http://localhost:4200", + # Next 3 entries are for proxmox-migration. Remove when completed. https://tacc-main.atlassian.net/browse/WG-615 + "production-tmp": "https://hazmapper.tacc.utexas.edu/hazmapper-tmp", + "staging-tmp": "https://hazmapper.tacc.utexas.edu/staging-tmp", + "dev-tmp": "https://hazmapper.tacc.utexas.edu/dev-tmp", + } + if settings.APP_ENV in hazmapper_urls: + return hazmapper_urls[settings.APP_ENV] + else: + logger.exception(f"Unknown/unsupported APP_ENV:{settings.APP_ENV}") + raise ApiException(f"Unknown APP_ENV:{settings.APP_ENV}") From 3bc04fd77efcdabcfba141f829532da7bcb013c0 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 28 Jul 2026 20:52:15 -0500 Subject: [PATCH 13/23] Fix comments --- geoapi/misc/README.md | 21 +++------------- geoapi/misc/dry_run_published_ds_maps.py | 23 ++---------------- geoapi/routes/published_ds_maps.py | 5 ++-- geoapi/services/tippecanoe.py | 5 ++-- geoapi/tasks/published_ds_maps.py | 24 +++++++------------ .../test_published_ds_maps_routes.py | 2 +- geoapi/utils/client_backend.py | 8 +------ 7 files changed, 20 insertions(+), 68 deletions(-) diff --git a/geoapi/misc/README.md b/geoapi/misc/README.md index 04137aaf..488508f1 100644 --- a/geoapi/misc/README.md +++ b/geoapi/misc/README.md @@ -1,22 +1,12 @@ # `misc/` — one-off operational scripts -Ad-hoc scripts run by hand (not part of the API or the Celery schedule). Today -there is just one. +Ad-hoc scripts run by hand (not part of the API or the Celery schedule). ## `dry_run_published_ds_maps.py` -A **non-destructive** dry run of the WG-703 published-DesignSafe-maps archive. It -runs the real selector → exporter → tiler against whatever database the -environment points at and prints: +A dry run creating the published-DesignSafe-maps archive. -- project / feature / COG-footprint counts and a per-`feature_type` breakdown, -- per-feature property-payload sizes (which keys are heaviest — informs whether to - split project-level fields into a companion `projects.json`), -- the archive bounds, and -- the **total archive size**. - -It writes the archive to `/assets/tmp/published_ds_maps.pmtiles` (a scratch path, -never `/assets/public/`); delete it when done. +It writes the archive to `/assets/tmp/published_ds_maps.pmtiles` ### Run against the local dev stack @@ -26,11 +16,6 @@ docker exec -it geoapi_workers python misc/dry_run_published_ds_maps.py ### Run against production -The deployed geoapi containers get their DB password, `APP_ENV`, etc. from a -secrets file on the host — see `Core-Portal-Deployments/geoapi-services/camino/docker-compose.yml`, -where `backend` and `celerybeat` use `env_file: /opt/portal/conf/secrets.env`. -Reuse that same file for a one-off container (org/tag come from `prod.env`): - ```bash docker run --rm \ --env-file /opt/portal/conf/secrets.env \ diff --git a/geoapi/misc/dry_run_published_ds_maps.py b/geoapi/misc/dry_run_published_ds_maps.py index 3586be1a..1a4e7e79 100644 --- a/geoapi/misc/dry_run_published_ds_maps.py +++ b/geoapi/misc/dry_run_published_ds_maps.py @@ -1,25 +1,6 @@ """ -Non-destructive dry run of the WG-703 published DesignSafe maps archive. - -Runs the REAL selector + exporter + tiler against whatever database the -environment is configured for, tiles into a scratch directory, and prints -counts, a per-layer breakdown, per-feature property-size stats, bounds, and the -total archive size. Use it to answer "how big is this archive and how many -projects/features?" before deploying -- e.g. pointed at production data. - -It is safe to run against production: - * DB access is READ-ONLY (SELECT via the ORM; nothing is written to the DB). - * Output goes ONLY to a scratch dir -- it never touches the real - assets/public area or manifest.json. - * It does NOT take the generation lock and does NOT enqueue anything. - -Requirements: this must run from a geoapi-workers image (it has tippecanoe + this -code) with the environment pointed at the target DB. APP_ENV must match the tier -whose maps you want (production maps are tagged deployment=production), because -the selector filters on `deployment == APP_ENV` and the URLs baked into the -archive come from APP_ENV. - -See misc/README.md for how to run it (local + production) and what it reports. +Non-destructive dry run of creating published DesignSafe maps archive. Useful for testing out +on prod and staging systems before deployment. See README.md """ import argparse diff --git a/geoapi/routes/published_ds_maps.py b/geoapi/routes/published_ds_maps.py index 581bf736..6ff5ec86 100644 --- a/geoapi/routes/published_ds_maps.py +++ b/geoapi/routes/published_ds_maps.py @@ -10,9 +10,8 @@ class PublishedDsMapsController(Controller): """Unauthenticated discovery endpoint for the published DesignSafe maps archive. - Cheap: it just serves the sidecar ``manifest.json`` written by the nightly - task -- it never streams the archive (nginx serves the .pmtiles directly) and - never builds it (a fresh environment 404s until the first run completes). + It just serves the sidecar ``manifest.json`` written by the nightly + task. It never streams the archive as nginx serves the .pmtiles directly). """ path = "/designsafe" diff --git a/geoapi/services/tippecanoe.py b/geoapi/services/tippecanoe.py index e03ecbb0..815e5eaf 100644 --- a/geoapi/services/tippecanoe.py +++ b/geoapi/services/tippecanoe.py @@ -138,8 +138,9 @@ def geojson_layers_to_pmtiles( str(min_zoom), "-z", str(max_zoom), - # --- no feature dropping, at any zoom (WG-703 requires every marker) --- - "--drop-rate=1", # keep every feature at every zoom (no rate thinning) + # --- no feature dropping, at any zoom (as used for all-published-DS-maps (WG-703) + # so it requires every marker + "--drop-rate=1", # keep every feature at every zoom "--no-feature-limit", # don't cap features per tile "--no-tile-size-limit", # don't drop to keep tiles under the size cap "--no-tiny-polygon-reduction", # keep small footprints as-is diff --git a/geoapi/tasks/published_ds_maps.py b/geoapi/tasks/published_ds_maps.py index 07552d7f..fe98c96f 100644 --- a/geoapi/tasks/published_ds_maps.py +++ b/geoapi/tasks/published_ds_maps.py @@ -23,12 +23,7 @@ from geoapi.utils.assets import get_temp_dir from geoapi.utils.client_backend import get_deployed_geoapi_url -# Layout of the shared public asset area (served unauthenticated by nginx). -PUBLIC_ASSET_SUBDIR = "public" -MANIFEST_FILENAME = "manifest.json" -# Versioned, lexicographically-sortable archive name: -# published_ds_maps_20260724T030000Z.pmtiles ARCHIVE_PREFIX = "published_ds_maps_" ARCHIVE_SUFFIX = ".pmtiles" @@ -36,22 +31,22 @@ SCHEMA_VERSION = 1 # Prune archives older than this, but never the one the manifest references. 26h -# (not 24h) leaves margin for a late run so a predecessor isn't removed early; in -# steady state this keeps two generations. +# (not 24h) leaves margin for a late run so a predecessor isn't removed early; so +# we should be keeping two generations. PRUNE_AGE_SECONDS = 26 * 3600 # Redis lock so a nightly beat, a startup enqueue, and a manual trigger can't run -# two tippecanoe builds at once. Auto-expires so a crashed run doesn't wedge it. +# two tippecanoe builds at once. GENERATION_LOCK_KEY = "published_ds_maps:generation" GENERATION_LOCK_TIMEOUT_SECONDS = 2 * 3600 def public_asset_dir() -> str: - return os.path.join(settings.ASSETS_BASE_DIR, PUBLIC_ASSET_SUBDIR) + return os.path.join(settings.ASSETS_BASE_DIR, "public") def manifest_path() -> str: - return os.path.join(public_asset_dir(), MANIFEST_FILENAME) + return os.path.join(public_asset_dir(), "manifest.json") def read_manifest() -> Optional[dict]: @@ -64,7 +59,7 @@ def read_manifest() -> Optional[dict]: def _public_asset_url(filename: str) -> str: - return f"{get_deployed_geoapi_url()}/assets/{PUBLIC_ASSET_SUBDIR}/{filename}" + return f"{get_deployed_geoapi_url()}/assets/public/{filename}" def _utc_now() -> datetime: @@ -72,7 +67,6 @@ def _utc_now() -> datetime: def _archive_filename(now: datetime) -> str: - # UTC, explicit Z, no colons/spaces -> filesystem- and URL-safe and sortable return f"{ARCHIVE_PREFIX}{now.strftime('%Y%m%dT%H%M%SZ')}{ARCHIVE_SUFFIX}" @@ -289,12 +283,10 @@ def _write_manifest(manifest: dict) -> None: def enqueue_generation_if_missing() -> bool: - """Cold-start self-heal: enqueue a build if no manifest exists yet. + """Enqueue a build if no manifest exists yet. Called on app startup so a fresh environment produces an archive within one - task run instead of waiting for the nightly beat. Never runs generation - inline (that would block the worker for minutes); the Redis lock keeps this - from overlapping the nightly or a manual trigger. + task run instead of waiting for the nightly beat. :return: True if a generation task was enqueued """ diff --git a/geoapi/tests/api_tests/test_published_ds_maps_routes.py b/geoapi/tests/api_tests/test_published_ds_maps_routes.py index 678fdb59..ac136dbc 100644 --- a/geoapi/tests/api_tests/test_published_ds_maps_routes.py +++ b/geoapi/tests/api_tests/test_published_ds_maps_routes.py @@ -21,7 +21,7 @@ def test_get_manifest_returns_200_when_present(test_client): assert resp.json() == manifest -def test_get_manifest_returns_404_on_cold_start(test_client): +def test_get_manifest_returns_404_when_missing(test_client): with patch("geoapi.routes.published_ds_maps.read_manifest", return_value=None): resp = test_client.get(ROUTE) assert resp.status_code == 404 diff --git a/geoapi/utils/client_backend.py b/geoapi/utils/client_backend.py index 3f529415..64d5a8d1 100644 --- a/geoapi/utils/client_backend.py +++ b/geoapi/utils/client_backend.py @@ -86,15 +86,9 @@ def get_deployed_hazmapper_url(): """ Get the Hazmapper frontend base URL for the current environment. - Mirrors the frontend's host + basePath convention (see hazmapper - getHazmapperBase/getBasePath): a single host, distinguished by a path - prefix per environment. Used to build public-map deep links, e.g. + Used to build public-map deep links, e.g. ``{base}/project-public/{uuid}?selectedFeature={id}``. """ - # basePath per environment mirrors the frontend's getBasePath. Note that - # production lives under /hazmapper (not root) -- this matches the canonical - # public-map links DesignSafe itself publishes - # (https://hazmapper.tacc.utexas.edu/hazmapper/project-public//). hazmapper_urls = { "local": "http://localhost:4200", "production": "https://hazmapper.tacc.utexas.edu/hazmapper", From 1487221e6e32f9c5a6f883b405f6917d79285509 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 28 Jul 2026 21:08:06 -0500 Subject: [PATCH 14/23] Fix imports and formatting --- geoapi/services/features.py | 2 +- geoapi/tests/api_tests/test_vectors_service.py | 2 +- geoapi/tests/conftest.py | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/geoapi/services/features.py b/geoapi/services/features.py index 30d846c5..27374f79 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -17,7 +17,7 @@ from geoapi.services.images import ImageService, ImageData from geoapi.services.tippecanoe import TippecanoeService from geoapi.services.vectors import VectorService, SUPPORTED_VECTOR_EXTENSIONS -from geoapi.models import Feature, FeatureAsset, Overlay, User, TileServer +from geoapi.models import Feature, FeatureAsset, User, TileServer from geoapi.exceptions import ( InvalidGeoJSON, ApiException, diff --git a/geoapi/tests/api_tests/test_vectors_service.py b/geoapi/tests/api_tests/test_vectors_service.py index 202dc855..14c339c4 100644 --- a/geoapi/tests/api_tests/test_vectors_service.py +++ b/geoapi/tests/api_tests/test_vectors_service.py @@ -6,7 +6,7 @@ import geopandas as gpd from shapely.geometry import Point -from geoapi.services.vectors import VectorService, SUPPORTED_VECTOR_EXTENSIONS +from geoapi.services.vectors import VectorService from geoapi.services.tippecanoe import TippecanoeService import pyogrio import pytest diff --git a/geoapi/tests/conftest.py b/geoapi/tests/conftest.py index 777b9f12..3bb87dc5 100644 --- a/geoapi/tests/conftest.py +++ b/geoapi/tests/conftest.py @@ -433,9 +433,7 @@ def shapefile_fixture(): @pytest.fixture(scope="function") def point_and_polygon_geojson_fixture(): home = os.path.dirname(__file__) - with open( - os.path.join(home, "fixtures/TACC_point_and_polygon.geojson"), "rb" - ) as f: + with open(os.path.join(home, "fixtures/TACC_point_and_polygon.geojson"), "rb") as f: yield FileStorage(f, filename="TACC_point_and_polygon.geojson") From 9c1b8c536a618d2b4590d5382e354de993293ce4 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 28 Jul 2026 21:11:42 -0500 Subject: [PATCH 15/23] Fix formatting --- geoapi/custom/designsafe/published_export.py | 6 +++-- geoapi/custom/designsafe/published_maps.py | 4 +--- geoapi/misc/dry_run_published_ds_maps.py | 4 +++- geoapi/tasks/published_ds_maps.py | 9 ++++--- .../custom/designsafe/test_published_maps.py | 24 +++++++++++-------- .../tasks_tests/test_published_ds_maps.py | 20 ++++++++++++---- 6 files changed, 42 insertions(+), 25 deletions(-) diff --git a/geoapi/custom/designsafe/published_export.py b/geoapi/custom/designsafe/published_export.py index 3cf7a167..2085ecb9 100644 --- a/geoapi/custom/designsafe/published_export.py +++ b/geoapi/custom/designsafe/published_export.py @@ -160,8 +160,10 @@ def _thumbnail_url(feature, geoapi_base) -> Optional[str]: served at ``{geoapi}/assets/{asset.path}``. """ for asset in feature.assets: - if asset.asset_type == "image" and asset.path and asset.path.endswith( - ".jpeg" + if ( + asset.asset_type == "image" + and asset.path + and asset.path.endswith(".jpeg") ): thumb_path = asset.path[: -len(".jpeg")] + ".thumb.jpeg" return f"{geoapi_base}/assets/{thumb_path}" diff --git a/geoapi/custom/designsafe/published_maps.py b/geoapi/custom/designsafe/published_maps.py index 03fd2b17..463f8309 100644 --- a/geoapi/custom/designsafe/published_maps.py +++ b/geoapi/custom/designsafe/published_maps.py @@ -107,9 +107,7 @@ def _fetch_base_project(cls, designsafe_project_id: str) -> dict: return detail.get("baseProject") or {} @classmethod - def get_published_maps( - cls, database_session - ) -> List[PublishedMap]: + def get_published_maps(cls, database_session) -> List[PublishedMap]: """Resolve the published, this-deployment, public maps to local Projects. :param database_session: SQLAlchemy session bound to this deployment's DB diff --git a/geoapi/misc/dry_run_published_ds_maps.py b/geoapi/misc/dry_run_published_ds_maps.py index 1a4e7e79..e2eee97a 100644 --- a/geoapi/misc/dry_run_published_ds_maps.py +++ b/geoapi/misc/dry_run_published_ds_maps.py @@ -51,7 +51,9 @@ def main(): print(f"DB host / name : {settings.DB_HOST} / {settings.DB_NAME}") print(f"geoapi base URL : {get_deployed_geoapi_url()}") print(f"hazmapper base URL : {get_deployed_hazmapper_url()}") - print(f"zoom range : z{PUBLISHED_DS_MAPS_MIN_ZOOM}-z{PUBLISHED_DS_MAPS_MAX_ZOOM}") + print( + f"zoom range : z{PUBLISHED_DS_MAPS_MIN_ZOOM}-z{PUBLISHED_DS_MAPS_MAX_ZOOM}" + ) print("-" * 72) with create_task_session() as session: diff --git a/geoapi/tasks/published_ds_maps.py b/geoapi/tasks/published_ds_maps.py index fe98c96f..102e33a6 100644 --- a/geoapi/tasks/published_ds_maps.py +++ b/geoapi/tasks/published_ds_maps.py @@ -23,7 +23,6 @@ from geoapi.utils.assets import get_temp_dir from geoapi.utils.client_backend import get_deployed_geoapi_url - ARCHIVE_PREFIX = "published_ds_maps_" ARCHIVE_SUFFIX = ".pmtiles" @@ -184,9 +183,7 @@ def generate_published_ds_maps_pmtiles(): geoapi/misc/dry_run_published_ds_maps.py. """ start_time = time.time() - redis_client = redis.Redis( - host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=0 - ) + redis_client = redis.Redis(host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=0) lock = redis_client.lock( GENERATION_LOCK_KEY, timeout=GENERATION_LOCK_TIMEOUT_SECONDS ) @@ -220,7 +217,9 @@ def generate_published_ds_maps_pmtiles(): os.makedirs(public_asset_dir(), exist_ok=True) # Work under ASSETS_BASE_DIR/tmp so the finished archive is on the SAME # filesystem as the public dir and os.replace() is atomic. - work_dir = tempfile.mkdtemp(prefix="published_ds_maps_", dir=str(get_temp_dir())) + work_dir = tempfile.mkdtemp( + prefix="published_ds_maps_", dir=str(get_temp_dir()) + ) now = _utc_now() filename = _archive_filename(now) try: diff --git a/geoapi/tests/custom/designsafe/test_published_maps.py b/geoapi/tests/custom/designsafe/test_published_maps.py index 1a755297..cc7a4f8a 100644 --- a/geoapi/tests/custom/designsafe/test_published_maps.py +++ b/geoapi/tests/custom/designsafe/test_published_maps.py @@ -28,7 +28,9 @@ def _get_json(url): offset = int(url.split("offset=")[1].split("&")[0]) return {"result": pages.get(offset, [])} designsafe_project_id = url.rsplit("/", 1)[1] - return {"baseProject": {"hazmapperMaps": details.get(designsafe_project_id, [])}} + return { + "baseProject": {"hazmapperMaps": details.get(designsafe_project_id, [])} + } return _get_json @@ -64,20 +66,20 @@ def test_get_published_maps_selects_only_matching_maps_and_logs_exclusions( } ], # excluded: resolves to a non-public project -> error logged - "PRJ-2": [ - {"uuid": str(private_project.uuid), "deployment": this_deployment} - ], + "PRJ-2": [{"uuid": str(private_project.uuid), "deployment": this_deployment}], # excluded: registered by a different deployment tier - "PRJ-3": [ - {"uuid": str(public_project.uuid), "deployment": "some-other-tier"} - ], + "PRJ-3": [{"uuid": str(public_project.uuid), "deployment": "some-other-tier"}], # excluded: this deployment, but no such project in the DB -> warning "PRJ-4": [{"uuid": missing_uuid, "deployment": this_deployment}], } with patch.object( - PublishedMapsService, "_get_json", side_effect=_fake_designsafe_api(pages, details) - ), patch.object(PublishedMapsService, "inter_request_sleep_seconds", 0), caplog.at_level( + PublishedMapsService, + "_get_json", + side_effect=_fake_designsafe_api(pages, details), + ), patch.object( + PublishedMapsService, "inter_request_sleep_seconds", 0 + ), caplog.at_level( logging.WARNING ): selected = PublishedMapsService.get_published_maps(db_session) @@ -115,7 +117,9 @@ def test_selector_returns_empty_when_no_deployment_matches(db_session): } with patch.object( - PublishedMapsService, "_get_json", side_effect=_fake_designsafe_api(pages, details) + PublishedMapsService, + "_get_json", + side_effect=_fake_designsafe_api(pages, details), ), patch.object(PublishedMapsService, "inter_request_sleep_seconds", 0): selected = PublishedMapsService.get_published_maps(db_session) diff --git a/geoapi/tests/tasks_tests/test_published_ds_maps.py b/geoapi/tests/tasks_tests/test_published_ds_maps.py index 5a0cb11b..2fca6010 100644 --- a/geoapi/tests/tasks_tests/test_published_ds_maps.py +++ b/geoapi/tests/tasks_tests/test_published_ds_maps.py @@ -24,7 +24,13 @@ "geometry": { "type": "Polygon", "coordinates": [ - [[-97.8, 30.2], [-97.6, 30.2], [-97.6, 30.4], [-97.8, 30.4], [-97.8, 30.2]] + [ + [-97.8, 30.2], + [-97.6, 30.2], + [-97.6, 30.4], + [-97.8, 30.4], + [-97.8, 30.2], + ] ], }, "properties": {"feature_type": "cog", "feature_id": None}, @@ -173,13 +179,17 @@ def _wrong_count(layers, output_path, *a, **k): def test_enqueue_generation_if_missing_enqueues_only_when_absent(): - with patch.object(published_ds_maps, "read_manifest", return_value={"url": "x"}), patch.object( + with patch.object( + published_ds_maps, "read_manifest", return_value={"url": "x"} + ), patch.object( published_ds_maps.generate_published_ds_maps_pmtiles, "delay" ) as delay: assert published_ds_maps.enqueue_generation_if_missing() is False delay.assert_not_called() - with patch.object(published_ds_maps, "read_manifest", return_value=None), patch.object( + with patch.object( + published_ds_maps, "read_manifest", return_value=None + ), patch.object( published_ds_maps.generate_published_ds_maps_pmtiles, "delay" ) as delay: assert published_ds_maps.enqueue_generation_if_missing() is True @@ -192,7 +202,9 @@ def test_generate_prunes_old_archives_but_keeps_recent(tmp_path): os.makedirs(public_dir, exist_ok=True) old = os.path.join(public_dir, f"{ARCHIVE_PREFIX}20200101T000000Z{ARCHIVE_SUFFIX}") - recent = os.path.join(public_dir, f"{ARCHIVE_PREFIX}20990101T000000Z{ARCHIVE_SUFFIX}") + recent = os.path.join( + public_dir, f"{ARCHIVE_PREFIX}20990101T000000Z{ARCHIVE_SUFFIX}" + ) for path in (old, recent): with open(path, "wb") as f: f.write(b"old") From 4a3148cae23d7f748e9e0afbbfa1c07010947250 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Wed, 29 Jul 2026 13:29:40 -0500 Subject: [PATCH 16/23] Tweak tiling setting --- geoapi/misc/dry_run_published_ds_maps.py | 38 +++++++++++++++--- geoapi/services/tippecanoe.py | 39 ++++++++++++------- .../tests/services/test_tippecanoe_service.py | 10 ++--- 3 files changed, 61 insertions(+), 26 deletions(-) diff --git a/geoapi/misc/dry_run_published_ds_maps.py b/geoapi/misc/dry_run_published_ds_maps.py index e2eee97a..e5567233 100644 --- a/geoapi/misc/dry_run_published_ds_maps.py +++ b/geoapi/misc/dry_run_published_ds_maps.py @@ -7,13 +7,16 @@ import json import os import shutil +import subprocess import tempfile +import time from collections import Counter, defaultdict from geoapi.custom.designsafe.published_export import PublishedMapsExportService from geoapi.custom.designsafe.published_maps import PublishedMapsService from geoapi.db import create_task_session from geoapi.services.tippecanoe import ( + PUBLISHED_DS_MAPS_BASE_ZOOM, PUBLISHED_DS_MAPS_MAX_ZOOM, PUBLISHED_DS_MAPS_MIN_ZOOM, TippecanoeService, @@ -52,7 +55,8 @@ def main(): print(f"geoapi base URL : {get_deployed_geoapi_url()}") print(f"hazmapper base URL : {get_deployed_hazmapper_url()}") print( - f"zoom range : z{PUBLISHED_DS_MAPS_MIN_ZOOM}-z{PUBLISHED_DS_MAPS_MAX_ZOOM}" + f"zoom range : z{PUBLISHED_DS_MAPS_MIN_ZOOM}-z{PUBLISHED_DS_MAPS_MAX_ZOOM} " + f"(all features kept at base z{PUBLISHED_DS_MAPS_BASE_ZOOM}+)" ) print("-" * 72) @@ -97,23 +101,45 @@ def main(): bounds = _bounds(features) print(f"\nBounds [minx,miny,maxx,maxy]: {bounds}") - # tile into a scratch path (NOT the real public area) + # tile into a scratch path (NOT the real public area). Tile each layer + # separately (with timing) and then tile-join into the final archive. archive_path = args.out or os.path.join( str(get_temp_dir()), "published_ds_maps.pmtiles" ) os.makedirs(os.path.dirname(archive_path), exist_ok=True) work_dir = tempfile.mkdtemp(prefix="layers_", dir=os.path.dirname(archive_path)) + written_total = 0 try: layer_files = _write_layer_files(features, work_dir) - print(f"\nTiling {len(layer_files)} layer(s) into {archive_path} ...") - written = TippecanoeService.geojson_layers_to_pmtiles(layer_files, archive_path) + print("\nPer-layer tiling (feature count / time / size):") + per_layer_pmtiles = [] + for layer_name, geojson_path in layer_files: + layer_pmtiles = os.path.join(work_dir, f"{layer_name}.pmtiles") + start = time.time() + count = TippecanoeService.geojson_layers_to_pmtiles( + [(layer_name, geojson_path)], layer_pmtiles + ) + elapsed = time.time() - start + written_total += count or 0 + per_layer_pmtiles.append(layer_pmtiles) + print( + f" {layer_name:<14} {count or 0:>7} feat {elapsed:>8.1f}s " + f"{_human_bytes(os.path.getsize(layer_pmtiles))}" + ) + + print(f"\nCombining {len(per_layer_pmtiles)} layer(s) -> {archive_path} ...") + subprocess.run( + ["tile-join", "--force", "--no-tile-size-limit", "-o", archive_path] + + per_layer_pmtiles, + check=True, + ) finally: shutil.rmtree(work_dir, ignore_errors=True) archive_size = os.path.getsize(archive_path) print("-" * 72) - print(f"tippecanoe wrote : {written} features (source had {stats['total']})") - if written != stats["total"]: + print(f"features tiled : {written_total} (source had {stats['total']})") + if written_total != stats["total"]: print(" WARNING: count mismatch -- features were dropped!") print(f"ARCHIVE SIZE : {_human_bytes(archive_size)} ({archive_size} bytes)") print(f"Archive path : {archive_path}") diff --git a/geoapi/services/tippecanoe.py b/geoapi/services/tippecanoe.py index 815e5eaf..c11e62f9 100644 --- a/geoapi/services/tippecanoe.py +++ b/geoapi/services/tippecanoe.py @@ -10,8 +10,15 @@ # tippecanoe binary; overridable for environments where it is not on PATH TIPPECANOE_BIN = os.environ.get("TIPPECANOE_BIN", "tippecanoe") -PUBLISHED_DS_MAPS_MIN_ZOOM = 6 -PUBLISHED_DS_MAPS_MAX_ZOOM = 16 +# Zoom config for the combined published DesignSafe maps archive (WG-703). +# min_zoom z2 -- world overview (features are thinned below base_zoom; lets +# the layer show a sparse scatter when zoomed out) +# base_zoom z8 -- every feature is kept at z8 and every zoom above (matches the +# ReconPortal's selectedEventZoomToLevel); below z8 they are thinned +# max_zoom z14 -- ~0.5m coordinate precision +PUBLISHED_DS_MAPS_MIN_ZOOM = 2 +PUBLISHED_DS_MAPS_BASE_ZOOM = 8 +PUBLISHED_DS_MAPS_MAX_ZOOM = 14 # Floor for tippecanoe's guessed maximum zoom (-zg). # @@ -102,25 +109,28 @@ def geojson_layers_to_pmtiles( layers: List[Tuple[str, str]], output_path: str, min_zoom: int = PUBLISHED_DS_MAPS_MIN_ZOOM, + base_zoom: int = PUBLISHED_DS_MAPS_BASE_ZOOM, max_zoom: int = PUBLISHED_DS_MAPS_MAX_ZOOM, ) -> Optional[int]: """ - Tile per-layer GeoJSON files into one PMTiles archive with NO dropping. + Tile per-layer GeoJSON files into one PMTiles archive. Used for the combined published DesignSafe maps archive (WG-703). Unlike - ``geojson_to_pmtiles`` (per-feature vector ingest, which guesses maxzoom - and may shed on tile overflow), this pins a moderate zoom range and - disables every form of feature thinning, so every marker is present at - every zoom. Deep display is handled by client overzoom -- see - ``PUBLISHED_DS_MAPS_MAX_ZOOM``. + ``geojson_to_pmtiles`` (per-feature vector ingest, which guesses maxzoom), + this pins a fixed zoom range and keeps every feature at ``base_zoom`` and + above (``-B``) -- so nothing is dropped where the consumer actually looks + (its selectedEventZoomToLevel) up through overzoom. Below ``base_zoom`` + features are thinned into a cheap sparse overview. Per-tile size/feature + limits are disabled so ``base_zoom``+ stays complete. Each entry becomes its own vector-tile layer, so the consumer can style and toggle by feature type. :param layers: list of ``(layer_name, geojson_path)`` :param output_path: path where the .pmtiles archive will be written - :param min_zoom: minimum zoom (default z6) - :param max_zoom: maximum zoom (default z16) + :param min_zoom: minimum zoom (default z2) + :param base_zoom: keep all features at/above this zoom; thin below (default z8) + :param max_zoom: maximum zoom (default z14) :return: the feature count tippecanoe reports writing (for the caller's source-vs-archive count check), or ``None`` if it can't be parsed :raises RuntimeError: if the tippecanoe binary is missing or exits non-zero @@ -138,10 +148,11 @@ def geojson_layers_to_pmtiles( str(min_zoom), "-z", str(max_zoom), - # --- no feature dropping, at any zoom (as used for all-published-DS-maps (WG-703) - # so it requires every marker - "--drop-rate=1", # keep every feature at every zoom - "--no-feature-limit", # don't cap features per tile + # keep every feature at base_zoom and above (no dropping where the + # consumer looks); below base_zoom, thin into a sparse overview + "-B", + str(base_zoom), + "--no-feature-limit", # don't cap features per tile (base_zoom+ complete) "--no-tile-size-limit", # don't drop to keep tiles under the size cap "--no-tiny-polygon-reduction", # keep small footprints as-is "--no-line-simplification", # don't move vertices (COG footprints stay put) diff --git a/geoapi/tests/services/test_tippecanoe_service.py b/geoapi/tests/services/test_tippecanoe_service.py index 4c09e585..11f9dce7 100644 --- a/geoapi/tests/services/test_tippecanoe_service.py +++ b/geoapi/tests/services/test_tippecanoe_service.py @@ -95,16 +95,14 @@ def test_geojson_layers_to_pmtiles_builds_no_drop_command(): cmd = mock_run.call_args[0][0] assert cmd[0] == "tippecanoe" assert "-o" in cmd and cmd[cmd.index("-o") + 1] == "/tmp/out.pmtiles" - # fixed moderate zoom range (overzoom handles deep display) - assert cmd[cmd.index("-Z") + 1] == "6" - assert cmd[cmd.index("-z") + 1] == "16" - # no dropping of any kind - assert "--drop-rate=1" in cmd + assert cmd[cmd.index("-Z") + 1] == "2" + assert cmd[cmd.index("-z") + 1] == "14" + assert cmd[cmd.index("-B") + 1] == "8" assert "--no-feature-limit" in cmd assert "--no-tile-size-limit" in cmd assert "--no-tiny-polygon-reduction" in cmd assert "--no-line-simplification" in cmd - # crucially NOT the overflow-shedding used by per-feature ingest + assert "--drop-rate=1" not in cmd assert "--drop-densest-as-needed" not in cmd # one named layer per entry assert "points:/tmp/points.geojson" in cmd From eebcbc782e8b745d3d675f053d5174968ced2db2 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Wed, 29 Jul 2026 14:19:13 -0500 Subject: [PATCH 17/23] Make leaner tiles via drop-densest and low-zoom simplification --- geoapi/services/tippecanoe.py | 24 +++++++++---------- .../tests/services/test_tippecanoe_service.py | 10 ++++---- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/geoapi/services/tippecanoe.py b/geoapi/services/tippecanoe.py index c11e62f9..690a0ea6 100644 --- a/geoapi/services/tippecanoe.py +++ b/geoapi/services/tippecanoe.py @@ -117,14 +117,15 @@ def geojson_layers_to_pmtiles( Used for the combined published DesignSafe maps archive (WG-703). Unlike ``geojson_to_pmtiles`` (per-feature vector ingest, which guesses maxzoom), - this pins a fixed zoom range and keeps every feature at ``base_zoom`` and - above (``-B``) -- so nothing is dropped where the consumer actually looks - (its selectedEventZoomToLevel) up through overzoom. Below ``base_zoom`` - features are thinned into a cheap sparse overview. Per-tile size/feature - limits are disabled so ``base_zoom``+ stays complete. + this pins a fixed zoom range and keeps features from ``base_zoom`` up + (``-B``), where the ReconPortal actually looks (its selectedEventZoomToLevel), + through overzoom. Below ``base_zoom`` they thin into a sparser + overview. Tiles that would blow past the size cap shed their densest + features at that zoom (``--drop-densest-as-needed``); those reappear at + higher zoom where the tile fits (so no feature is lost from the archive). - Each entry becomes its own vector-tile layer, so the consumer can style - and toggle by feature type. + Each entry becomes its own vector-tile layer, so the consumer (i.e. ReconPortal) + can style and toggle by feature type. :param layers: list of ``(layer_name, geojson_path)`` :param output_path: path where the .pmtiles archive will be written @@ -148,14 +149,11 @@ def geojson_layers_to_pmtiles( str(min_zoom), "-z", str(max_zoom), - # keep every feature at base_zoom and above (no dropping where the - # consumer looks); below base_zoom, thin into a sparse overview "-B", str(base_zoom), - "--no-feature-limit", # don't cap features per tile (base_zoom+ complete) - "--no-tile-size-limit", # don't drop to keep tiles under the size cap - "--no-tiny-polygon-reduction", # keep small footprints as-is - "--no-line-simplification", # don't move vertices (COG footprints stay put) + "--drop-densest-as-needed", + "--no-feature-limit", + "--no-tiny-polygon-reduction", ] for layer_name, geojson_path in layers: # `-L name:file` reads file into its own named vector-tile layer diff --git a/geoapi/tests/services/test_tippecanoe_service.py b/geoapi/tests/services/test_tippecanoe_service.py index 11f9dce7..4f52bf69 100644 --- a/geoapi/tests/services/test_tippecanoe_service.py +++ b/geoapi/tests/services/test_tippecanoe_service.py @@ -82,7 +82,7 @@ def _point_counts_by_zoom(pmtiles_path): return counts -def test_geojson_layers_to_pmtiles_builds_no_drop_command(): +def test_geojson_layers_to_pmtiles_builds_command(): with patch("geoapi.services.tippecanoe.subprocess.run") as mock_run: mock_run.return_value = _completed(stderr="42 features, 1234 bytes of geometry") count = TippecanoeService.geojson_layers_to_pmtiles( @@ -98,13 +98,11 @@ def test_geojson_layers_to_pmtiles_builds_no_drop_command(): assert cmd[cmd.index("-Z") + 1] == "2" assert cmd[cmd.index("-z") + 1] == "14" assert cmd[cmd.index("-B") + 1] == "8" + assert "--drop-densest-as-needed" in cmd + assert "--no-tile-size-limit" not in cmd assert "--no-feature-limit" in cmd - assert "--no-tile-size-limit" in cmd assert "--no-tiny-polygon-reduction" in cmd - assert "--no-line-simplification" in cmd assert "--drop-rate=1" not in cmd - assert "--drop-densest-as-needed" not in cmd - # one named layer per entry assert "points:/tmp/points.geojson" in cmd assert "cog:/tmp/cog.geojson" in cmd @@ -121,7 +119,7 @@ def test_parse_written_feature_count_ignores_progress_line(): @pytest.mark.worker -def test_geojson_layers_to_pmtiles_no_drop_multilayer(): +def test_geojson_layers_to_pmtiles_multilayer(): # tile a point layer and a polygon (COG-footprint-like) layer into one # archive; assert the reported count matches input and both layers survive. out_dir = tempfile.mkdtemp(prefix="geoapi_published_ds_maps_test_") From e8fbcdee3610b50bb00565fc203268a716776d83 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 30 Jul 2026 09:55:26 -0500 Subject: [PATCH 18/23] slim feature props; move COG footprints to cogs.geojson --- geoapi/custom/designsafe/published_export.py | 111 ++++++++---------- geoapi/misc/dry_run_published_ds_maps.py | 30 +++-- geoapi/tasks/published_ds_maps.py | 91 ++++++++++---- .../designsafe/test_published_export.py | 53 +++++---- .../tasks_tests/test_published_ds_maps.py | 61 ++++++++-- 5 files changed, 214 insertions(+), 132 deletions(-) diff --git a/geoapi/custom/designsafe/published_export.py b/geoapi/custom/designsafe/published_export.py index 2085ecb9..7aa4b077 100644 --- a/geoapi/custom/designsafe/published_export.py +++ b/geoapi/custom/designsafe/published_export.py @@ -18,6 +18,8 @@ "designsafe.storage.published/{designsafe_project_id}" ) +COG_LARGE_EXTENT_WARN_DEG = 30 + # asset_type (FeatureAsset) -> feature_type carried in the archive. Types not # listed here fall through to a geometry-based point/shape classification (this # covers plain geometry features and PMTiles "vector" assets, whose own detail @@ -32,42 +34,44 @@ class PublishedMapsExportService: - """Turn selected published maps into GeoJSON features for tiling. + """Turn selected published maps into GeoJSON for the public archive. - Emits one GeoJSON feature per Hazmapper feature, plus one footprint polygon - per internal COG (rasters are never tiled into this archive -- only their - bounds, carrying a layer descriptor the consumer can use to load the real - COG on demand). Every feature carries a small, stable, snake_case property - set that WG-671 (Recon Portal) depends on. + Emits per-Hazmapper-feature dicts for the tiled PMTiles archive, plus + internal-COG footprint polygons for a companion cogs.geojson to be rendered + client-side. Every feature carries a property set that Recon Portal uses. """ @classmethod def build_features( cls, database_session, published_maps: List[PublishedMap] - ) -> Tuple[List[dict], dict]: - """Build the GeoJSON features and summary stats. - - :return: ``(features, stats)`` where ``features`` is a list of GeoJSON - Feature dicts and ``stats`` has ``feature_count`` (DB features), - ``cog_count`` (COG footprints), ``project_count`` and ``total``. + ) -> Tuple[List[dict], List[dict], dict]: + """Build the tiled features and the COG footprints. + + :return: ``(features, cog_features, stats)``. ``features`` are the + per-Hazmapper-feature dicts tiled into the PMTiles archive. + ``cog_features`` are internal-COG footprint polygons written to a + companion cogs.geojson and rendered client-side (not tiled -- see + _cog_footprint). ``stats`` has ``feature_count`` (tiled), + ``cog_count``, ``project_count`` and ``total`` (tiled). """ geoapi_base = get_deployed_geoapi_url() hazmapper_base = get_deployed_hazmapper_url() features: List[dict] = [] + cog_features: List[dict] = [] feature_count = 0 cog_count = 0 for published in published_maps: project = published.project common = { - "project_uuid": str(project.uuid), - "project_name": project.name, - "ds_project_id": published.designsafe_project_id, - "ds_doi": published.designsafe_doi, + "hazmapper_project_uuid": str(project.uuid), + "hazmapper_project_name": project.name, + "ds_project_name": published.designsafe_project_title, "ds_project_url": DESIGNSAFE_PUBLISHED_BROWSER_URL.format( designsafe_project_id=published.designsafe_project_id ), + "hazmapper_url": cls._hazmapper_url(hazmapper_base, project.uuid), } # regular Hazmapper features @@ -90,28 +94,16 @@ def build_features( { "feature_id": feature.id, "feature_type": cls._feature_type(feature), - "hazmapper_url": cls._hazmapper_url( - hazmapper_base, project.uuid, feature.id - ), - "has_assets": bool(feature.assets), - "created_date": ( - feature.created_date.isoformat() - if feature.created_date - else None - ), } ) - thumbnail_url = cls._thumbnail_url(feature, geoapi_base) - if thumbnail_url: - properties["thumbnail_url"] = thumbnail_url features.append( {"type": "Feature", "geometry": geometry, "properties": properties} ) feature_count += 1 - # internal COG footprints (never the pixels -- just the outline + a - # descriptor the consumer uses to instantiate the real tile layer) + # internal COG footprints -> companion cogs.geojson (rendered + # client-side, not tiled), each carrying a layer descriptor + tile URL for tile_server in ( database_session.query(TileServer) .filter(TileServer.project_id == project.id) @@ -119,11 +111,9 @@ def build_features( .filter(TileServer.kind == "cog") .all() ): - footprint = cls._cog_footprint( - tile_server, common, hazmapper_base, geoapi_base, project.uuid - ) + footprint = cls._cog_footprint(tile_server, common, geoapi_base) if footprint: - features.append(footprint) + cog_features.append(footprint) cog_count += 1 stats = { @@ -133,14 +123,13 @@ def build_features( "total": len(features), } logger.info( - "Public export built %s feature(s): %s DB features + %s COG footprints " + "Public export built %s tiled feature(s) + %s COG footprint(s) " "across %s project(s).", - stats["total"], feature_count, cog_count, stats["project_count"], ) - return features, stats + return features, cog_features, stats @staticmethod def _feature_type(feature) -> str: @@ -152,34 +141,11 @@ def _feature_type(feature) -> str: return "point" if "point" in geometry_type else "shape" @staticmethod - def _thumbnail_url(feature, geoapi_base) -> Optional[str]: - """URL of an image feature's thumbnail, if it has one. - - Image assets are stored as ``.jpeg`` alongside a - ``.thumb.jpeg`` thumbnail (see FeaturesService.fromImage), and are - served at ``{geoapi}/assets/{asset.path}``. - """ - for asset in feature.assets: - if ( - asset.asset_type == "image" - and asset.path - and asset.path.endswith(".jpeg") - ): - thumb_path = asset.path[: -len(".jpeg")] + ".thumb.jpeg" - return f"{geoapi_base}/assets/{thumb_path}" - return None - - @staticmethod - def _hazmapper_url(hazmapper_base, project_uuid, feature_id=None) -> str: - url = f"{hazmapper_base}/project-public/{project_uuid}" - if feature_id is not None: - url += f"?selectedFeature={feature_id}" - return url + def _hazmapper_url(hazmapper_base, project_uuid) -> str: + return f"{hazmapper_base}/project-public/{project_uuid}" @classmethod - def _cog_footprint( - cls, tile_server, common, hazmapper_base, geoapi_base, project_uuid - ) -> Optional[dict]: + def _cog_footprint(cls, tile_server, common, geoapi_base) -> Optional[dict]: tile_options = tile_server.tileOptions or {} bounds = tile_options.get("bounds") # bounds is leaflet-style [[minLat, minLng], [maxLat, maxLng]] @@ -191,6 +157,7 @@ def _cog_footprint( ) return None (south, west), (north, east) = bounds + # Filled footprint polygon. COGs go into a companion cogs.geojson ring = [ [west, south], [east, south], @@ -199,6 +166,22 @@ def _cog_footprint( [west, south], ] geometry = {"type": "Polygon", "coordinates": [ring]} + width_deg, height_deg = abs(east - west), abs(north - south) + if max(width_deg, height_deg) > COG_LARGE_EXTENT_WARN_DEG: + logger.warning( + "COG %s footprint extent %.2f x %.2f deg is unusually large " + "(possible georef error); still included.", + tile_server.id, + width_deg, + height_deg, + ) + else: + logger.info( + "COG %s footprint extent %.3f x %.3f deg.", + tile_server.id, + width_deg, + height_deg, + ) ui_options = tile_server.uiOptions or {} render_options = ui_options.get("renderOptions") or {} @@ -225,8 +208,6 @@ def _cog_footprint( { "feature_id": None, "feature_type": "cog", - "hazmapper_url": cls._hazmapper_url(hazmapper_base, project_uuid), - "has_assets": True, "tile_server_id": tile_server.id, "layer_name": tile_server.name, "layer_type": tile_server.type, diff --git a/geoapi/misc/dry_run_published_ds_maps.py b/geoapi/misc/dry_run_published_ds_maps.py index e5567233..c687515b 100644 --- a/geoapi/misc/dry_run_published_ds_maps.py +++ b/geoapi/misc/dry_run_published_ds_maps.py @@ -69,11 +69,12 @@ def main(): "matches the maps' deployment tag and that the DB is the right tier." ) return - features, stats = PublishedMapsExportService.build_features(session, published) + features, cog_features, stats = PublishedMapsExportService.build_features( + session, published + ) - print(f"DB features : {stats['feature_count']}") - print(f"COG footprints : {stats['cog_count']}") - print(f"Total features : {stats['total']}") + print(f"Tiled features : {stats['feature_count']}") + print(f"COG footprints : {stats['cog_count']} (companion cogs.geojson)") # per-layer (feature_type) breakdown by_type = Counter(f["properties"].get("feature_type") for f in features) @@ -98,11 +99,12 @@ def main(): for key, nbytes in sorted(per_key_bytes.items(), key=lambda kv: -kv[1])[:8]: print(f" {key:<16} {_human_bytes(nbytes)}") - bounds = _bounds(features) + bounds = _bounds(features + cog_features) print(f"\nBounds [minx,miny,maxx,maxy]: {bounds}") # tile into a scratch path (NOT the real public area). Tile each layer - # separately (with timing) and then tile-join into the final archive. + # separately (with timing) and then tile-join into the final archive. COGs are + # NOT tiled -- they go to a companion cogs.geojson (see below). archive_path = args.out or os.path.join( str(get_temp_dir()), "published_ds_maps.pmtiles" ) @@ -136,16 +138,24 @@ def main(): finally: shutil.rmtree(work_dir, ignore_errors=True) + # companion cogs.geojson (COG footprints, rendered client-side -- not tiled) + cogs_path = os.path.splitext(archive_path)[0] + ".cogs.geojson" + with open(cogs_path, "w") as f: + json.dump({"type": "FeatureCollection", "features": cog_features}, f) + cogs_size = os.path.getsize(cogs_path) + archive_size = os.path.getsize(archive_path) print("-" * 72) - print(f"features tiled : {written_total} (source had {stats['total']})") - if written_total != stats["total"]: + print(f"features tiled : {written_total} (source had {stats['feature_count']})") + if written_total != stats["feature_count"]: print(" WARNING: count mismatch -- features were dropped!") print(f"ARCHIVE SIZE : {_human_bytes(archive_size)} ({archive_size} bytes)") + print(f"cogs.geojson : {_human_bytes(cogs_size)} ({stats['cog_count']} COGs)") print(f"Archive path : {archive_path}") + print(f"COGs path : {cogs_path}") print( - f"\n(Delete {archive_path} when done. Nothing was written to the real " - "assets/public area or manifest.)" + f"\n(Delete {archive_path} and {cogs_path} when done. Nothing was written " + "to the real assets/public area or manifest.)" ) diff --git a/geoapi/tasks/published_ds_maps.py b/geoapi/tasks/published_ds_maps.py index 102e33a6..3d7f666d 100644 --- a/geoapi/tasks/published_ds_maps.py +++ b/geoapi/tasks/published_ds_maps.py @@ -23,16 +23,23 @@ from geoapi.utils.assets import get_temp_dir from geoapi.utils.client_backend import get_deployed_geoapi_url +# Main pmtiles archive that will contain all published ds map data in one file +# (with one exception, see COGS_SUFFIX below) ARCHIVE_PREFIX = "published_ds_maps_" ARCHIVE_SUFFIX = ".pmtiles" +# Companion GeoJSON of internal-COG footprints, rendered client-side (NOT tiled): +# a COG's extent can be large (up to worldwide), and tiling a filled polygon that +# big fills every tile it covers and blows up the build. Client-side GeoJSON draws +# it for free at any size. +COGS_SUFFIX = ".cogs.geojson" # Bumped when the manifest/property schema changes in a way consumers must notice. SCHEMA_VERSION = 1 -# Prune archives older than this, but never the one the manifest references. 26h +# Prune archives older than this, but never the one the manifest references. 30h # (not 24h) leaves margin for a late run so a predecessor isn't removed early; so # we should be keeping two generations. -PRUNE_AGE_SECONDS = 26 * 3600 +PRUNE_AGE_SECONDS = 30 * 3600 # Redis lock so a nightly beat, a startup enqueue, and a manual trigger can't run # two tippecanoe builds at once. @@ -65,8 +72,16 @@ def _utc_now() -> datetime: return datetime.now(timezone.utc) +def _timestamp(now: datetime) -> str: + return now.strftime("%Y%m%dT%H%M%SZ") + + def _archive_filename(now: datetime) -> str: - return f"{ARCHIVE_PREFIX}{now.strftime('%Y%m%dT%H%M%SZ')}{ARCHIVE_SUFFIX}" + return f"{ARCHIVE_PREFIX}{_timestamp(now)}{ARCHIVE_SUFFIX}" + + +def _cogs_filename(now: datetime) -> str: + return f"{ARCHIVE_PREFIX}{_timestamp(now)}{COGS_SUFFIX}" def _write_layer_files(features: List[dict], work_dir: str) -> List[Tuple[str, str]]: @@ -129,16 +144,22 @@ def _verify_feature_count(source_count: int, written_count: Optional[int]) -> No logger.info("Public archive feature count verified: %s.", source_count) -def _prune_old_archives(keep_filename: str) -> None: - """Delete archives older than PRUNE_AGE_SECONDS, never the referenced one.""" +def _prune_old_public_files(keep_filenames: set) -> None: + """Delete versioned archive/cogs files older than PRUNE_AGE_SECONDS, never the + just-written ones or the ones the manifest currently references.""" directory = public_asset_dir() manifest = read_manifest() or {} - referenced = os.path.basename(manifest.get("url", "")) if manifest else "" + keep = set(keep_filenames) | { + os.path.basename(manifest.get("url", "")), + os.path.basename(manifest.get("cogs_url", "")), + } now = time.time() for name in os.listdir(directory): - if not (name.startswith(ARCHIVE_PREFIX) and name.endswith(ARCHIVE_SUFFIX)): + if not name.startswith(ARCHIVE_PREFIX): continue - if name == keep_filename or name == referenced: + if not (name.endswith(ARCHIVE_SUFFIX) or name.endswith(COGS_SUFFIX)): + continue + if name in keep: continue path = os.path.join(directory, name) try: @@ -149,21 +170,22 @@ def _prune_old_archives(keep_filename: str) -> None: try: os.remove(path) logger.info( - "Pruned stale public archive %s (age %.1fh).", name, age / 3600 + "Pruned stale public file %s (age %.1fh).", name, age / 3600 ) except OSError: - logger.exception("Failed to prune public archive %s.", name) + logger.exception("Failed to prune public file %s.", name) @app.task(queue="heavy") def generate_published_ds_maps_pmtiles(): """Nightly: build the combined public PMTiles archive for published DS maps. - Selects this deployment's published, public maps, exports their features - (plus internal-COG footprints) to GeoJSON, tiles them with no dropping, and - atomically publishes the archive + a sidecar manifest into the shared public - asset area. Fails loudly and leaves the previous archive/manifest untouched - on any error. Guarded by a Redis lock so runs can't overlap. + Selects this deployment's published, public maps, tiles their features into + the PMTiles archive, writes internal-COG footprints to a companion + cogs.geojson (rendered client-side, not tiled), and atomically publishes the + archive + cogs.geojson + a sidecar manifest into the shared public asset area. + Fails loudly and leaves the previous files untouched on any error. Guarded by + a Redis lock so runs can't overlap. Runs nightly via celery beat; on a fresh environment app startup also enqueues one build if no manifest exists yet (see on_startup in app.py). To regenerate @@ -203,7 +225,7 @@ def generate_published_ds_maps_pmtiles(): settings.APP_ENV, ) return - features, stats = PublishedMapsExportService.build_features( + features, cog_features, stats = PublishedMapsExportService.build_features( session, published_maps ) @@ -215,13 +237,14 @@ def generate_published_ds_maps_pmtiles(): return os.makedirs(public_asset_dir(), exist_ok=True) - # Work under ASSETS_BASE_DIR/tmp so the finished archive is on the SAME + # Work under ASSETS_BASE_DIR/tmp so the finished files are on the SAME # filesystem as the public dir and os.replace() is atomic. work_dir = tempfile.mkdtemp( prefix="published_ds_maps_", dir=str(get_temp_dir()) ) now = _utc_now() filename = _archive_filename(now) + cogs_filename = _cogs_filename(now) try: layer_files = _write_layer_files(features, work_dir) staged_archive = os.path.join(work_dir, filename) @@ -230,32 +253,41 @@ def generate_published_ds_maps_pmtiles(): ) _verify_feature_count(len(features), written_count) - final_archive = os.path.join(public_asset_dir(), filename) - # atomic within the same filesystem: a partial archive is never readable - os.replace(staged_archive, final_archive) + # atomic within the same filesystem: a partial file is never readable + os.replace(staged_archive, os.path.join(public_asset_dir(), filename)) + # companion COG footprints -- rendered client-side, not tiled + _write_public_file( + {"type": "FeatureCollection", "features": cog_features}, + cogs_filename, + work_dir, + ) finally: shutil.rmtree(work_dir, ignore_errors=True) manifest = { "url": _public_asset_url(filename), + "cogs_url": _public_asset_url(cogs_filename), "generated_at": now.isoformat(), "feature_count": stats["total"], + "cog_count": stats["cog_count"], "project_count": stats["project_count"], - "bounds": _bounds(features), + "bounds": _bounds(features + cog_features), "zoom_min": PUBLISHED_DS_MAPS_MIN_ZOOM, "zoom_max": PUBLISHED_DS_MAPS_MAX_ZOOM, "schema_version": SCHEMA_VERSION, } _write_manifest(manifest) - # prune only after the manifest points at the new archive - _prune_old_archives(keep_filename=filename) + # prune only after the manifest points at the new files + _prune_old_public_files({filename, cogs_filename}) logger.info( - "Public pmtiles archive published: %s (%s features across %s projects) " - "in %.1fs.", + "Public archive published: %s (%s features) + %s (%s COGs) across " + "%s projects in %.1fs.", filename, stats["total"], + cogs_filename, + stats["cog_count"], stats["project_count"], time.time() - start_time, ) @@ -281,6 +313,15 @@ def _write_manifest(manifest: dict) -> None: os.replace(tmp_path, manifest_path()) +def _write_public_file(data: dict, filename: str, work_dir: str) -> None: + """Atomically place a JSON file into the public asset dir. Staged in work_dir + (same filesystem as the public dir), so os.replace() is atomic.""" + staged = os.path.join(work_dir, filename) + with open(staged, "w") as f: + json.dump(data, f) + os.replace(staged, os.path.join(public_asset_dir(), filename)) + + def enqueue_generation_if_missing() -> bool: """Enqueue a build if no manifest exists yet. diff --git a/geoapi/tests/custom/designsafe/test_published_export.py b/geoapi/tests/custom/designsafe/test_published_export.py index f7485ac1..b7068d4c 100644 --- a/geoapi/tests/custom/designsafe/test_published_export.py +++ b/geoapi/tests/custom/designsafe/test_published_export.py @@ -82,47 +82,59 @@ def test_build_features_emits_features_cog_and_enriched_properties(db_session): image = _image_feature(db_session, project) _cog_tile_server(db_session, project) - features, stats = PublishedMapsExportService.build_features( + features, cog_features, stats = PublishedMapsExportService.build_features( db_session, [_published(project)] ) + # COGs are split out (companion cogs.geojson), not tiled with the features assert stats == { "feature_count": 2, "cog_count": 1, "project_count": 1, - "total": 3, + "total": 2, } - assert len(features) == 3 + assert len(features) == 2 # point + image, tiled + assert len(cog_features) == 1 # COG footprint -> cogs.geojson by_type = {} for f in features: by_type.setdefault(f["properties"]["feature_type"], []).append(f) - # every feature carries the shared publication context - for f in features: + # every emitted feature (tiled + COG) carries the shared project context + for f in features + cog_features: props = f["properties"] - assert props["project_uuid"] == str(project.uuid) - assert props["ds_project_id"] == "PRJ-1" + assert props["hazmapper_project_uuid"] == str(project.uuid) + assert props["hazmapper_project_name"] == "pub" + assert props["ds_project_name"] == "Published One" assert "designsafe.storage.published/PRJ-1" in props["ds_project_url"] - assert props["ds_doi"] == "10.17603/ds2-test" - assert "/project-public/" in props["hazmapper_url"] - - # point feature: geometry-derived type, deep link, no thumbnail + # project map link only (dedup-able) -- NOT a per-feature deep link + assert props["hazmapper_url"].endswith(f"/project-public/{project.uuid}") + assert "selectedFeature" not in props["hazmapper_url"] + # slimmed-out fields are gone + for gone in ( + "project_uuid", + "ds_project_id", + "ds_doi", + "has_assets", + "created_date", + ): + assert gone not in props + + # point feature: geometry-derived type; feature_id lets the consumer build the + # deep link as f"{hazmapper_url}?selectedFeature={feature_id}" pt = by_type["point"][0] assert pt["properties"]["feature_id"] == point.id - assert pt["properties"]["has_assets"] is False - assert f"selectedFeature={point.id}" in pt["properties"]["hazmapper_url"] assert "thumbnail_url" not in pt["properties"] - # image feature: asset-derived type + thumbnail pointer + # image feature: asset-derived type; no baked thumbnail (fetched on click) img = by_type["image"][0] assert img["properties"]["feature_id"] == image.id - assert img["properties"]["has_assets"] is True - assert img["properties"]["thumbnail_url"].endswith(".thumb.jpeg") - assert "/assets/" in img["properties"]["thumbnail_url"] + assert "thumbnail_url" not in img["properties"] - # cog footprint: polygon + resolvable tile url + field-for-field descriptor - cog = by_type["cog"][0] + # COG footprint: a filled polygon (rendered client-side from cogs.geojson) + + # resolvable tile url + field-for-field descriptor + cog = cog_features[0] + assert cog["properties"]["feature_type"] == "cog" assert cog["geometry"]["type"] == "Polygon" assert cog["properties"]["feature_id"] is None assert cog["properties"]["layer_name"] == "my-cog" @@ -157,9 +169,10 @@ def test_external_tile_layers_are_not_exported(db_session): db_session.add(external) db_session.commit() - features, stats = PublishedMapsExportService.build_features( + features, cog_features, stats = PublishedMapsExportService.build_features( db_session, [_published(project)] ) assert stats["cog_count"] == 0 assert features == [] + assert cog_features == [] diff --git a/geoapi/tests/tasks_tests/test_published_ds_maps.py b/geoapi/tests/tasks_tests/test_published_ds_maps.py index 2fca6010..562881a6 100644 --- a/geoapi/tests/tasks_tests/test_published_ds_maps.py +++ b/geoapi/tests/tasks_tests/test_published_ds_maps.py @@ -10,9 +10,11 @@ from geoapi.tasks.published_ds_maps import ( ARCHIVE_PREFIX, ARCHIVE_SUFFIX, + COGS_SUFFIX, generate_published_ds_maps_pmtiles, ) +# tiled features (points/shapes) -- COGs are NOT tiled; they go to cogs.geojson FEATURES = [ { "type": "Feature", @@ -33,16 +35,35 @@ ] ], }, + "properties": {"feature_type": "shape", "feature_id": 2}, + }, +] +# COG footprint -> companion cogs.geojson (extends the western bound to -98.0) +COG_FEATURES = [ + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-98.0, 30.0], + [-97.9, 30.0], + [-97.9, 30.1], + [-98.0, 30.1], + [-98.0, 30.0], + ] + ], + }, "properties": {"feature_type": "cog", "feature_id": None}, }, ] -STATS = {"feature_count": 1, "cog_count": 1, "project_count": 1, "total": 2} +STATS = {"feature_count": 2, "cog_count": 1, "project_count": 1, "total": 2} def _write_dummy_archive(layers, output_path, *args, **kwargs): with open(output_path, "wb") as f: f.write(b"PMTILES-DUMMY") - return len(FEATURES) # tippecanoe "wrote" 2 features + return len(FEATURES) # tippecanoe "wrote" the 2 tiled features class _ContextManagerMock: @@ -56,7 +77,7 @@ def __exit__(self, *exc): def _patchers( isolated_assets_dir, published_maps=("sentinel",), - build_return=(FEATURES, STATS), + build_return=(FEATURES, COG_FEATURES, STATS), tile_side_effect=_write_dummy_archive, lock_acquired=True, ): @@ -112,20 +133,30 @@ def test_generate_writes_archive_and_manifest(tmp_path): for n in os.listdir(public_dir) if n.startswith(ARCHIVE_PREFIX) and n.endswith(ARCHIVE_SUFFIX) ] + cogs = [n for n in os.listdir(public_dir) if n.endswith(COGS_SUFFIX)] assert len(archives) == 1 - filename = archives[0] + assert len(cogs) == 1 + filename, cogs_filename = archives[0], cogs[0] + + # cogs.geojson is a FeatureCollection of the COG footprints + with open(os.path.join(public_dir, cogs_filename)) as f: + cogs_fc = json.load(f) + assert cogs_fc["type"] == "FeatureCollection" + assert len(cogs_fc["features"]) == 1 with open(os.path.join(public_dir, "manifest.json")) as f: manifest = json.load(f) assert manifest["feature_count"] == 2 + assert manifest["cog_count"] == 1 assert manifest["project_count"] == 1 - assert manifest["zoom_min"] == 6 - assert manifest["zoom_max"] == 16 + assert manifest["zoom_min"] == 2 + assert manifest["zoom_max"] == 14 assert manifest["schema_version"] == 1 assert manifest["url"].endswith(f"/assets/public/{filename}") + assert manifest["cogs_url"].endswith(f"/assets/public/{cogs_filename}") assert len(manifest["bounds"]) == 4 - # bounds span both features - assert manifest["bounds"][0] == pytest.approx(-97.8) + # bounds span the tiled features AND the COG footprint (COG extends west to -98) + assert manifest["bounds"][0] == pytest.approx(-98.0) assert manifest["bounds"][2] == pytest.approx(-97.6) @@ -173,7 +204,8 @@ def _wrong_count(layers, output_path, *a, **k): placed = [ n for n in os.listdir(public_dir) - if n.startswith(ARCHIVE_PREFIX) and n.endswith(ARCHIVE_SUFFIX) + if n.startswith(ARCHIVE_PREFIX) + and (n.endswith(ARCHIVE_SUFFIX) or n.endswith(COGS_SUFFIX)) ] assert placed == [] @@ -202,21 +234,26 @@ def test_generate_prunes_old_archives_but_keeps_recent(tmp_path): os.makedirs(public_dir, exist_ok=True) old = os.path.join(public_dir, f"{ARCHIVE_PREFIX}20200101T000000Z{ARCHIVE_SUFFIX}") + old_cogs = os.path.join( + public_dir, f"{ARCHIVE_PREFIX}20200101T000000Z{COGS_SUFFIX}" + ) recent = os.path.join( public_dir, f"{ARCHIVE_PREFIX}20990101T000000Z{ARCHIVE_SUFFIX}" ) - for path in (old, recent): + for path in (old, old_cogs, recent): with open(path, "wb") as f: f.write(b"old") - # make `old` older than the 26h window; leave `recent` fresh + # make the old files older than the 26h window; leave `recent` fresh stale = time.time() - (30 * 3600) os.utime(old, (stale, stale)) + os.utime(old_cogs, (stale, stale)) patchers, _ = _patchers(assets_dir) _run(patchers) remaining = set(os.listdir(public_dir)) - assert os.path.basename(old) not in remaining # pruned (stale) + assert os.path.basename(old) not in remaining # stale .pmtiles pruned + assert os.path.basename(old_cogs) not in remaining # stale .cogs.geojson pruned assert os.path.basename(recent) in remaining # kept (fresh) # the newly generated archive is present too assert any( From 0e04d9db1852cf8694fcbb9cf999364fba0684c0 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 30 Jul 2026 11:59:10 -0500 Subject: [PATCH 19/23] Move COG/vector layers to companion geojson; slim tile props --- geoapi/custom/designsafe/published_export.py | 140 ++++++++++-------- geoapi/misc/dry_run_published_ds_maps.py | 35 +++-- geoapi/tasks/published_ds_maps.py | 53 ++++--- .../designsafe/test_published_export.py | 92 ++++++------ .../tasks_tests/test_published_ds_maps.py | 67 ++++++--- 5 files changed, 222 insertions(+), 165 deletions(-) diff --git a/geoapi/custom/designsafe/published_export.py b/geoapi/custom/designsafe/published_export.py index 7aa4b077..89e0d09a 100644 --- a/geoapi/custom/designsafe/published_export.py +++ b/geoapi/custom/designsafe/published_export.py @@ -1,4 +1,4 @@ -import json +import shapely.geometry from typing import List, Optional, Tuple from urllib.parse import quote @@ -18,12 +18,12 @@ "designsafe.storage.published/{designsafe_project_id}" ) -COG_LARGE_EXTENT_WARN_DEG = 30 +# A footprint wider/taller than this many degrees is logged as a likely georef error. +LARGE_EXTENT_WARN_DEG = 30 -# asset_type (FeatureAsset) -> feature_type carried in the archive. Types not -# listed here fall through to a geometry-based point/shape classification (this -# covers plain geometry features and PMTiles "vector" assets, whose own detail -# lives in a separate archive). +# asset_type (FeatureAsset) -> feature_type for tiled features. "vector" is handled +# separately as a layer footprint; other unlisted types fall through to a +# geometry-based point/shape classification. _ASSET_TYPE_TO_FEATURE_TYPE = { "image": "image", "video": "video", @@ -36,31 +36,33 @@ class PublishedMapsExportService: """Turn selected published maps into GeoJSON for the public archive. - Emits per-Hazmapper-feature dicts for the tiled PMTiles archive, plus - internal-COG footprint polygons for a companion cogs.geojson to be rendered - client-side. Every feature carries a property set that Recon Portal uses. + Emits per-Hazmapper-feature dicts for the tiled PMTiles archive, plus layer + footprints (internal COGs and PMTiles-vector uploads) for a companion GeoJSON + rendered client-side. Every feature carries a property set that Recon Portal + uses. """ @classmethod def build_features( cls, database_session, published_maps: List[PublishedMap] ) -> Tuple[List[dict], List[dict], dict]: - """Build the tiled features and the COG footprints. + """Build the tiled features and the layer footprints. - :return: ``(features, cog_features, stats)``. ``features`` are the + :return: ``(features, layer_features, stats)``. ``features`` are the per-Hazmapper-feature dicts tiled into the PMTiles archive. - ``cog_features`` are internal-COG footprint polygons written to a - companion cogs.geojson and rendered client-side (not tiled -- see - _cog_footprint). ``stats`` has ``feature_count`` (tiled), - ``cog_count``, ``project_count`` and ``total`` (tiled). + ``layer_features`` are footprints for internal COGs and PMTiles-vector + uploads, written to the companion GeoJSON and rendered client-side. + ``stats`` has ``feature_count`` (tiled), ``cog_count``, + ``vector_count``, ``project_count`` and ``total`` (tiled). """ geoapi_base = get_deployed_geoapi_url() hazmapper_base = get_deployed_hazmapper_url() features: List[dict] = [] - cog_features: List[dict] = [] + layer_features: List[dict] = [] feature_count = 0 cog_count = 0 + vector_count = 0 for published in published_maps: project = published.project @@ -74,7 +76,6 @@ def build_features( "hazmapper_url": cls._hazmapper_url(hazmapper_base, project.uuid), } - # regular Hazmapper features for feature in ( database_session.query(Feature) .filter(Feature.project_id == project.id) @@ -89,6 +90,21 @@ def build_features( continue if not geometry: continue + + # a PMTiles-vector upload -> layer footprint (pointer to its own + # tiles), not tiled here + vector_asset = next( + (a for a in feature.assets if a.asset_type == "vector"), None + ) + if vector_asset: + layer_features.append( + cls._vector_footprint( + feature, geometry, vector_asset, common, geoapi_base + ) + ) + vector_count += 1 + continue + properties = dict(common) properties.update( { @@ -96,14 +112,11 @@ def build_features( "feature_type": cls._feature_type(feature), } ) - features.append( {"type": "Feature", "geometry": geometry, "properties": properties} ) feature_count += 1 - # internal COG footprints -> companion cogs.geojson (rendered - # client-side, not tiled), each carrying a layer descriptor + tile URL for tile_server in ( database_session.query(TileServer) .filter(TileServer.project_id == project.id) @@ -113,23 +126,25 @@ def build_features( ): footprint = cls._cog_footprint(tile_server, common, geoapi_base) if footprint: - cog_features.append(footprint) + layer_features.append(footprint) cog_count += 1 stats = { "feature_count": feature_count, "cog_count": cog_count, + "vector_count": vector_count, "project_count": len(published_maps), "total": len(features), } logger.info( - "Public export built %s tiled feature(s) + %s COG footprint(s) " - "across %s project(s).", + "Public export built %s tiled feature(s) + %s COG + %s vector " + "footprint(s) across %s project(s).", feature_count, cog_count, + vector_count, stats["project_count"], ) - return features, cog_features, stats + return features, layer_features, stats @staticmethod def _feature_type(feature) -> str: @@ -157,7 +172,6 @@ def _cog_footprint(cls, tile_server, common, geoapi_base) -> Optional[dict]: ) return None (south, west), (north, east) = bounds - # Filled footprint polygon. COGs go into a companion cogs.geojson ring = [ [west, south], [east, south], @@ -166,30 +180,9 @@ def _cog_footprint(cls, tile_server, common, geoapi_base) -> Optional[dict]: [west, south], ] geometry = {"type": "Polygon", "coordinates": [ring]} - width_deg, height_deg = abs(east - west), abs(north - south) - if max(width_deg, height_deg) > COG_LARGE_EXTENT_WARN_DEG: - logger.warning( - "COG %s footprint extent %.2f x %.2f deg is unusually large " - "(possible georef error); still included.", - tile_server.id, - width_deg, - height_deg, - ) - else: - logger.info( - "COG %s footprint extent %.3f x %.3f deg.", - tile_server.id, - width_deg, - height_deg, - ) + cls._warn_if_large("COG", tile_server.id, abs(east - west), abs(north - south)) - ui_options = tile_server.uiOptions or {} - render_options = ui_options.get("renderOptions") or {} - - # Field-for-field mirror of hazmapper's TileServerLayer, JSON-encoded so - # the consumer can reconstruct the layer with no translation shim. (Vector - # tile attributes must be scalars, so this rides as a string; the flat - # fields below are convenience duplicates for quick styling/labeling.) + # TileServerLayer descriptor the consumer feeds to its layer builder. descriptor = { "id": tile_server.id, "name": tile_server.name, @@ -200,7 +193,7 @@ def _cog_footprint(cls, tile_server, common, geoapi_base) -> Optional[dict]: "url": tile_server.url, "attribution": tile_server.attribution, "tileOptions": tile_options, - "uiOptions": ui_options, + "uiOptions": tile_server.uiOptions or {}, } properties = dict(common) @@ -208,21 +201,50 @@ def _cog_footprint(cls, tile_server, common, geoapi_base) -> Optional[dict]: { "feature_id": None, "feature_type": "cog", - "tile_server_id": tile_server.id, - "layer_name": tile_server.name, - "layer_type": tile_server.type, - "min_zoom": tile_options.get("minZoom"), - "max_zoom": tile_options.get("maxZoom"), - "max_native_zoom": tile_options.get("maxNativeZoom"), - "attribution": tile_server.attribution, - "opacity": ui_options.get("opacity"), - "colormap_name": render_options.get("colormap_name"), "cog_url": cls._build_cog_url(tile_server, geoapi_base), - "tile_layer": json.dumps(descriptor), + "bounds": [west, south, east, north], + "tile_layer": descriptor, } ) return {"type": "Feature", "geometry": geometry, "properties": properties} + @classmethod + def _vector_footprint(cls, feature, geometry, vector_asset, common, geoapi_base): + """Layer footprint for a PMTiles-vector upload: the feature bbox + a pointer + to the upload's own PMTiles archive.""" + minx, miny, maxx, maxy = shapely.geometry.shape(geometry).bounds + cls._warn_if_large("Vector", feature.id, abs(maxx - minx), abs(maxy - miny)) + properties = dict(common) + properties.update( + { + "feature_id": feature.id, + "feature_type": "vector", + "pmtiles_url": f"{geoapi_base}/assets/{vector_asset.path}", + "bounds": [minx, miny, maxx, maxy], + } + ) + return {"type": "Feature", "geometry": geometry, "properties": properties} + + @staticmethod + def _warn_if_large(kind, obj_id, width_deg, height_deg): + if max(width_deg, height_deg) > LARGE_EXTENT_WARN_DEG: + logger.warning( + "%s %s footprint extent %.2f x %.2f deg is unusually large " + "(possible georef error); still included.", + kind, + obj_id, + width_deg, + height_deg, + ) + else: + logger.info( + "%s %s footprint extent %.3f x %.3f deg.", + kind, + obj_id, + width_deg, + height_deg, + ) + @staticmethod def _build_cog_url(tile_server, geoapi_base) -> str: """Pre-resolved TiTiler tile URL template for an internal COG. diff --git a/geoapi/misc/dry_run_published_ds_maps.py b/geoapi/misc/dry_run_published_ds_maps.py index c687515b..354636f4 100644 --- a/geoapi/misc/dry_run_published_ds_maps.py +++ b/geoapi/misc/dry_run_published_ds_maps.py @@ -69,12 +69,15 @@ def main(): "matches the maps' deployment tag and that the DB is the right tier." ) return - features, cog_features, stats = PublishedMapsExportService.build_features( - session, published + features, layer_features, stats = ( + PublishedMapsExportService.build_features(session, published) ) print(f"Tiled features : {stats['feature_count']}") - print(f"COG footprints : {stats['cog_count']} (companion cogs.geojson)") + print( + f"Layer footprints : {stats['cog_count']} COG + {stats['vector_count']} " + "vector (companion geojson)" + ) # per-layer (feature_type) breakdown by_type = Counter(f["properties"].get("feature_type") for f in features) @@ -99,12 +102,11 @@ def main(): for key, nbytes in sorted(per_key_bytes.items(), key=lambda kv: -kv[1])[:8]: print(f" {key:<16} {_human_bytes(nbytes)}") - bounds = _bounds(features + cog_features) + bounds = _bounds(features + layer_features) print(f"\nBounds [minx,miny,maxx,maxy]: {bounds}") - # tile into a scratch path (NOT the real public area). Tile each layer - # separately (with timing) and then tile-join into the final archive. COGs are - # NOT tiled -- they go to a companion cogs.geojson (see below). + # tile into a scratch path (NOT the real public area). Layer footprints (COGs + + # vectors) are not tiled -- they go to a companion geojson (see below). archive_path = args.out or os.path.join( str(get_temp_dir()), "published_ds_maps.pmtiles" ) @@ -138,11 +140,11 @@ def main(): finally: shutil.rmtree(work_dir, ignore_errors=True) - # companion cogs.geojson (COG footprints, rendered client-side -- not tiled) - cogs_path = os.path.splitext(archive_path)[0] + ".cogs.geojson" - with open(cogs_path, "w") as f: - json.dump({"type": "FeatureCollection", "features": cog_features}, f) - cogs_size = os.path.getsize(cogs_path) + # companion layer footprints (COGs + vectors), rendered client-side, not tiled + layers_path = os.path.splitext(archive_path)[0] + ".vectors_and_internal_cogs.geojson" + with open(layers_path, "w") as f: + json.dump({"type": "FeatureCollection", "features": layer_features}, f) + layers_size = os.path.getsize(layers_path) archive_size = os.path.getsize(archive_path) print("-" * 72) @@ -150,11 +152,14 @@ def main(): if written_total != stats["feature_count"]: print(" WARNING: count mismatch -- features were dropped!") print(f"ARCHIVE SIZE : {_human_bytes(archive_size)} ({archive_size} bytes)") - print(f"cogs.geojson : {_human_bytes(cogs_size)} ({stats['cog_count']} COGs)") + print( + f"layers geojson : {_human_bytes(layers_size)} " + f"({stats['cog_count']} COG + {stats['vector_count']} vector)" + ) print(f"Archive path : {archive_path}") - print(f"COGs path : {cogs_path}") + print(f"Layers path : {layers_path}") print( - f"\n(Delete {archive_path} and {cogs_path} when done. Nothing was written " + f"\n(Delete {archive_path} and {layers_path} when done. Nothing was written " "to the real assets/public area or manifest.)" ) diff --git a/geoapi/tasks/published_ds_maps.py b/geoapi/tasks/published_ds_maps.py index 3d7f666d..30633d6c 100644 --- a/geoapi/tasks/published_ds_maps.py +++ b/geoapi/tasks/published_ds_maps.py @@ -24,14 +24,12 @@ from geoapi.utils.client_backend import get_deployed_geoapi_url # Main pmtiles archive that will contain all published ds map data in one file -# (with one exception, see COGS_SUFFIX below) +# (COG and PMTiles-vector layers are the exception, see LAYERS_SUFFIX below) ARCHIVE_PREFIX = "published_ds_maps_" ARCHIVE_SUFFIX = ".pmtiles" -# Companion GeoJSON of internal-COG footprints, rendered client-side (NOT tiled): -# a COG's extent can be large (up to worldwide), and tiling a filled polygon that -# big fills every tile it covers and blows up the build. Client-side GeoJSON draws -# it for free at any size. -COGS_SUFFIX = ".cogs.geojson" +# Companion GeoJSON of layer footprints (internal COGs and PMTiles-vector uploads), +# rendered client-side rather than tiled. +LAYERS_SUFFIX = ".vectors_and_internal_cogs.geojson" # Bumped when the manifest/property schema changes in a way consumers must notice. SCHEMA_VERSION = 1 @@ -80,8 +78,8 @@ def _archive_filename(now: datetime) -> str: return f"{ARCHIVE_PREFIX}{_timestamp(now)}{ARCHIVE_SUFFIX}" -def _cogs_filename(now: datetime) -> str: - return f"{ARCHIVE_PREFIX}{_timestamp(now)}{COGS_SUFFIX}" +def _layers_filename(now: datetime) -> str: + return f"{ARCHIVE_PREFIX}{_timestamp(now)}{LAYERS_SUFFIX}" def _write_layer_files(features: List[dict], work_dir: str) -> List[Tuple[str, str]]: @@ -151,13 +149,13 @@ def _prune_old_public_files(keep_filenames: set) -> None: manifest = read_manifest() or {} keep = set(keep_filenames) | { os.path.basename(manifest.get("url", "")), - os.path.basename(manifest.get("cogs_url", "")), + os.path.basename(manifest.get("layers_url", "")), } now = time.time() for name in os.listdir(directory): if not name.startswith(ARCHIVE_PREFIX): continue - if not (name.endswith(ARCHIVE_SUFFIX) or name.endswith(COGS_SUFFIX)): + if not (name.endswith(ARCHIVE_SUFFIX) or name.endswith(LAYERS_SUFFIX)): continue if name in keep: continue @@ -181,11 +179,11 @@ def generate_published_ds_maps_pmtiles(): """Nightly: build the combined public PMTiles archive for published DS maps. Selects this deployment's published, public maps, tiles their features into - the PMTiles archive, writes internal-COG footprints to a companion - cogs.geojson (rendered client-side, not tiled), and atomically publishes the - archive + cogs.geojson + a sidecar manifest into the shared public asset area. - Fails loudly and leaves the previous files untouched on any error. Guarded by - a Redis lock so runs can't overlap. + the PMTiles archive, writes layer footprints (internal COGs and PMTiles-vector + uploads) to a companion GeoJSON rendered client-side, and atomically publishes + the archive + companion GeoJSON + a sidecar manifest into the shared public + asset area. Fails loudly and leaves the previous files untouched on any error. + Guarded by a Redis lock so runs can't overlap. Runs nightly via celery beat; on a fresh environment app startup also enqueues one build if no manifest exists yet (see on_startup in app.py). To regenerate @@ -225,8 +223,8 @@ def generate_published_ds_maps_pmtiles(): settings.APP_ENV, ) return - features, cog_features, stats = PublishedMapsExportService.build_features( - session, published_maps + features, layer_features, stats = ( + PublishedMapsExportService.build_features(session, published_maps) ) if not features: @@ -244,7 +242,7 @@ def generate_published_ds_maps_pmtiles(): ) now = _utc_now() filename = _archive_filename(now) - cogs_filename = _cogs_filename(now) + layers_filename = _layers_filename(now) try: layer_files = _write_layer_files(features, work_dir) staged_archive = os.path.join(work_dir, filename) @@ -255,10 +253,9 @@ def generate_published_ds_maps_pmtiles(): # atomic within the same filesystem: a partial file is never readable os.replace(staged_archive, os.path.join(public_asset_dir(), filename)) - # companion COG footprints -- rendered client-side, not tiled _write_public_file( - {"type": "FeatureCollection", "features": cog_features}, - cogs_filename, + {"type": "FeatureCollection", "features": layer_features}, + layers_filename, work_dir, ) finally: @@ -266,12 +263,13 @@ def generate_published_ds_maps_pmtiles(): manifest = { "url": _public_asset_url(filename), - "cogs_url": _public_asset_url(cogs_filename), + "layers_url": _public_asset_url(layers_filename), "generated_at": now.isoformat(), "feature_count": stats["total"], "cog_count": stats["cog_count"], + "vector_count": stats["vector_count"], "project_count": stats["project_count"], - "bounds": _bounds(features + cog_features), + "bounds": _bounds(features + layer_features), "zoom_min": PUBLISHED_DS_MAPS_MIN_ZOOM, "zoom_max": PUBLISHED_DS_MAPS_MAX_ZOOM, "schema_version": SCHEMA_VERSION, @@ -279,15 +277,16 @@ def generate_published_ds_maps_pmtiles(): _write_manifest(manifest) # prune only after the manifest points at the new files - _prune_old_public_files({filename, cogs_filename}) + _prune_old_public_files({filename, layers_filename}) logger.info( - "Public archive published: %s (%s features) + %s (%s COGs) across " - "%s projects in %.1fs.", + "Public archive published: %s (%s features) + %s (%s COGs, %s vectors) " + "across %s projects in %.1fs.", filename, stats["total"], - cogs_filename, + layers_filename, stats["cog_count"], + stats["vector_count"], stats["project_count"], time.time() - start_time, ) diff --git a/geoapi/tests/custom/designsafe/test_published_export.py b/geoapi/tests/custom/designsafe/test_published_export.py index b7068d4c..64660c5f 100644 --- a/geoapi/tests/custom/designsafe/test_published_export.py +++ b/geoapi/tests/custom/designsafe/test_published_export.py @@ -1,8 +1,7 @@ -import json import uuid from geoalchemy2.shape import from_shape -from shapely.geometry import Point +from shapely.geometry import Point, box from geoapi.models import Feature, FeatureAsset, Project, TileServer from geoapi.custom.designsafe.published_export import PublishedMapsExportService @@ -40,6 +39,22 @@ def _image_feature(db_session, project): return feature +def _vector_feature(db_session, project): + feature = Feature(project_id=project.id, properties={}) + feature.the_geom = from_shape(box(-99.0, 29.0, -98.0, 30.0), srid=4326) + asset_uuid = uuid.uuid4() + asset = FeatureAsset( + uuid=asset_uuid, + asset_type="vector", + path=f"{project.id}/{asset_uuid}.pmtiles", + feature=feature, + ) + feature.assets.append(asset) + db_session.add(feature) + db_session.commit() + return feature + + def _cog_tile_server(db_session, project): ts = TileServer( project_id=project.id, @@ -76,81 +91,73 @@ def _published(project, doi="10.17603/ds2-test"): ) -def test_build_features_emits_features_cog_and_enriched_properties(db_session): +def test_build_features_tiles_markers_and_lists_layer_footprints(db_session): project = _project(db_session) point = _point_feature(db_session, project) image = _image_feature(db_session, project) + vector = _vector_feature(db_session, project) _cog_tile_server(db_session, project) - features, cog_features, stats = PublishedMapsExportService.build_features( + features, layer_features, stats = PublishedMapsExportService.build_features( db_session, [_published(project)] ) - # COGs are split out (companion cogs.geojson), not tiled with the features + # markers are tiled; COG + vector become layer footprints (companion geojson) assert stats == { "feature_count": 2, "cog_count": 1, + "vector_count": 1, "project_count": 1, "total": 2, } - assert len(features) == 2 # point + image, tiled - assert len(cog_features) == 1 # COG footprint -> cogs.geojson + assert len(features) == 2 + assert len(layer_features) == 2 by_type = {} for f in features: by_type.setdefault(f["properties"]["feature_type"], []).append(f) + layers_by_type = {} + for f in layer_features: + layers_by_type.setdefault(f["properties"]["feature_type"], []).append(f) - # every emitted feature (tiled + COG) carries the shared project context - for f in features + cog_features: + # shared project context on everything (tiled + layer footprints) + for f in features + layer_features: props = f["properties"] assert props["hazmapper_project_uuid"] == str(project.uuid) assert props["hazmapper_project_name"] == "pub" assert props["ds_project_name"] == "Published One" assert "designsafe.storage.published/PRJ-1" in props["ds_project_url"] - # project map link only (dedup-able) -- NOT a per-feature deep link assert props["hazmapper_url"].endswith(f"/project-public/{project.uuid}") assert "selectedFeature" not in props["hazmapper_url"] - # slimmed-out fields are gone - for gone in ( - "project_uuid", - "ds_project_id", - "ds_doi", - "has_assets", - "created_date", - ): + for gone in ("project_uuid", "ds_project_id", "ds_doi", "has_assets", + "created_date"): assert gone not in props - # point feature: geometry-derived type; feature_id lets the consumer build the - # deep link as f"{hazmapper_url}?selectedFeature={feature_id}" - pt = by_type["point"][0] - assert pt["properties"]["feature_id"] == point.id - assert "thumbnail_url" not in pt["properties"] - - # image feature: asset-derived type; no baked thumbnail (fetched on click) - img = by_type["image"][0] - assert img["properties"]["feature_id"] == image.id - assert "thumbnail_url" not in img["properties"] - - # COG footprint: a filled polygon (rendered client-side from cogs.geojson) + - # resolvable tile url + field-for-field descriptor - cog = cog_features[0] - assert cog["properties"]["feature_type"] == "cog" + assert by_type["point"][0]["properties"]["feature_id"] == point.id + assert "thumbnail_url" not in by_type["point"][0]["properties"] + assert by_type["image"][0]["properties"]["feature_id"] == image.id + + # COG footprint: filled polygon + tile url + descriptor object + bounds + cog = layers_by_type["cog"][0] assert cog["geometry"]["type"] == "Polygon" assert cog["properties"]["feature_id"] is None - assert cog["properties"]["layer_name"] == "my-cog" - assert cog["properties"]["max_zoom"] == 22 - assert cog["properties"]["colormap_name"] == "terrain" + assert len(cog["properties"]["bounds"]) == 4 cog_url = cog["properties"]["cog_url"] assert "/tiles/cog/tiles/WebMercatorQuad/{z}/{x}/{y}.png?url=" in cog_url - assert "file%3A%2F%2F" in cog_url # url-encoded file:// assert "colormap_name=terrain" in cog_url - - descriptor = json.loads(cog["properties"]["tile_layer"]) + descriptor = cog["properties"]["tile_layer"] assert descriptor["internal"] is True assert descriptor["kind"] == "cog" - assert descriptor["type"] == "xyz" assert descriptor["tileOptions"]["maxZoom"] == 22 + # vector footprint: bbox polygon + pointer to its own PMTiles + vec = layers_by_type["vector"][0] + assert vec["geometry"]["type"] == "Polygon" + assert vec["properties"]["feature_id"] == vector.id + assert vec["properties"]["pmtiles_url"].endswith(".pmtiles") + assert "/assets/" in vec["properties"]["pmtiles_url"] + assert len(vec["properties"]["bounds"]) == 4 + def test_external_tile_layers_are_not_exported(db_session): project = _project(db_session) @@ -169,10 +176,11 @@ def test_external_tile_layers_are_not_exported(db_session): db_session.add(external) db_session.commit() - features, cog_features, stats = PublishedMapsExportService.build_features( + features, layer_features, stats = PublishedMapsExportService.build_features( db_session, [_published(project)] ) assert stats["cog_count"] == 0 + assert stats["vector_count"] == 0 assert features == [] - assert cog_features == [] + assert layer_features == [] diff --git a/geoapi/tests/tasks_tests/test_published_ds_maps.py b/geoapi/tests/tasks_tests/test_published_ds_maps.py index 562881a6..4d32625e 100644 --- a/geoapi/tests/tasks_tests/test_published_ds_maps.py +++ b/geoapi/tests/tasks_tests/test_published_ds_maps.py @@ -10,11 +10,11 @@ from geoapi.tasks.published_ds_maps import ( ARCHIVE_PREFIX, ARCHIVE_SUFFIX, - COGS_SUFFIX, + LAYERS_SUFFIX, generate_published_ds_maps_pmtiles, ) -# tiled features (points/shapes) -- COGs are NOT tiled; they go to cogs.geojson +# tiled features (points/shapes); layer footprints go to the companion geojson FEATURES = [ { "type": "Feature", @@ -38,8 +38,8 @@ "properties": {"feature_type": "shape", "feature_id": 2}, }, ] -# COG footprint -> companion cogs.geojson (extends the western bound to -98.0) -COG_FEATURES = [ +# layer footprints -> companion geojson (COG extends the western bound to -98.0) +LAYER_FEATURES = [ { "type": "Feature", "geometry": { @@ -56,8 +56,30 @@ }, "properties": {"feature_type": "cog", "feature_id": None}, }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-97.9, 30.0], + [-97.8, 30.0], + [-97.8, 30.1], + [-97.9, 30.1], + [-97.9, 30.0], + ] + ], + }, + "properties": {"feature_type": "vector", "feature_id": 3}, + }, ] -STATS = {"feature_count": 2, "cog_count": 1, "project_count": 1, "total": 2} +STATS = { + "feature_count": 2, + "cog_count": 1, + "vector_count": 1, + "project_count": 1, + "total": 2, +} def _write_dummy_archive(layers, output_path, *args, **kwargs): @@ -77,7 +99,7 @@ def __exit__(self, *exc): def _patchers( isolated_assets_dir, published_maps=("sentinel",), - build_return=(FEATURES, COG_FEATURES, STATS), + build_return=(FEATURES, LAYER_FEATURES, STATS), tile_side_effect=_write_dummy_archive, lock_acquired=True, ): @@ -133,27 +155,28 @@ def test_generate_writes_archive_and_manifest(tmp_path): for n in os.listdir(public_dir) if n.startswith(ARCHIVE_PREFIX) and n.endswith(ARCHIVE_SUFFIX) ] - cogs = [n for n in os.listdir(public_dir) if n.endswith(COGS_SUFFIX)] + layers = [n for n in os.listdir(public_dir) if n.endswith(LAYERS_SUFFIX)] assert len(archives) == 1 - assert len(cogs) == 1 - filename, cogs_filename = archives[0], cogs[0] + assert len(layers) == 1 + filename, layers_filename = archives[0], layers[0] - # cogs.geojson is a FeatureCollection of the COG footprints - with open(os.path.join(public_dir, cogs_filename)) as f: - cogs_fc = json.load(f) - assert cogs_fc["type"] == "FeatureCollection" - assert len(cogs_fc["features"]) == 1 + # the companion geojson is a FeatureCollection of the layer footprints + with open(os.path.join(public_dir, layers_filename)) as f: + layers_fc = json.load(f) + assert layers_fc["type"] == "FeatureCollection" + assert len(layers_fc["features"]) == 2 with open(os.path.join(public_dir, "manifest.json")) as f: manifest = json.load(f) assert manifest["feature_count"] == 2 assert manifest["cog_count"] == 1 + assert manifest["vector_count"] == 1 assert manifest["project_count"] == 1 assert manifest["zoom_min"] == 2 assert manifest["zoom_max"] == 14 assert manifest["schema_version"] == 1 assert manifest["url"].endswith(f"/assets/public/{filename}") - assert manifest["cogs_url"].endswith(f"/assets/public/{cogs_filename}") + assert manifest["layers_url"].endswith(f"/assets/public/{layers_filename}") assert len(manifest["bounds"]) == 4 # bounds span the tiled features AND the COG footprint (COG extends west to -98) assert manifest["bounds"][0] == pytest.approx(-98.0) @@ -205,7 +228,7 @@ def _wrong_count(layers, output_path, *a, **k): n for n in os.listdir(public_dir) if n.startswith(ARCHIVE_PREFIX) - and (n.endswith(ARCHIVE_SUFFIX) or n.endswith(COGS_SUFFIX)) + and (n.endswith(ARCHIVE_SUFFIX) or n.endswith(LAYERS_SUFFIX)) ] assert placed == [] @@ -234,26 +257,26 @@ def test_generate_prunes_old_archives_but_keeps_recent(tmp_path): os.makedirs(public_dir, exist_ok=True) old = os.path.join(public_dir, f"{ARCHIVE_PREFIX}20200101T000000Z{ARCHIVE_SUFFIX}") - old_cogs = os.path.join( - public_dir, f"{ARCHIVE_PREFIX}20200101T000000Z{COGS_SUFFIX}" + old_layers = os.path.join( + public_dir, f"{ARCHIVE_PREFIX}20200101T000000Z{LAYERS_SUFFIX}" ) recent = os.path.join( public_dir, f"{ARCHIVE_PREFIX}20990101T000000Z{ARCHIVE_SUFFIX}" ) - for path in (old, old_cogs, recent): + for path in (old, old_layers, recent): with open(path, "wb") as f: f.write(b"old") - # make the old files older than the 26h window; leave `recent` fresh + # make the old files older than the prune window; leave `recent` fresh stale = time.time() - (30 * 3600) os.utime(old, (stale, stale)) - os.utime(old_cogs, (stale, stale)) + os.utime(old_layers, (stale, stale)) patchers, _ = _patchers(assets_dir) _run(patchers) remaining = set(os.listdir(public_dir)) assert os.path.basename(old) not in remaining # stale .pmtiles pruned - assert os.path.basename(old_cogs) not in remaining # stale .cogs.geojson pruned + assert os.path.basename(old_layers) not in remaining # stale companion pruned assert os.path.basename(recent) in remaining # kept (fresh) # the newly generated archive is present too assert any( From a2f114be9fb0b6012015b1ec1291802f8087ea3f Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 31 Jul 2026 14:22:54 -0500 Subject: [PATCH 20/23] Fix formatting --- geoapi/misc/dry_run_published_ds_maps.py | 8 +++++--- geoapi/tasks/published_ds_maps.py | 4 ++-- geoapi/tests/custom/designsafe/test_published_export.py | 9 +++++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/geoapi/misc/dry_run_published_ds_maps.py b/geoapi/misc/dry_run_published_ds_maps.py index 354636f4..7beac929 100644 --- a/geoapi/misc/dry_run_published_ds_maps.py +++ b/geoapi/misc/dry_run_published_ds_maps.py @@ -69,8 +69,8 @@ def main(): "matches the maps' deployment tag and that the DB is the right tier." ) return - features, layer_features, stats = ( - PublishedMapsExportService.build_features(session, published) + features, layer_features, stats = PublishedMapsExportService.build_features( + session, published ) print(f"Tiled features : {stats['feature_count']}") @@ -141,7 +141,9 @@ def main(): shutil.rmtree(work_dir, ignore_errors=True) # companion layer footprints (COGs + vectors), rendered client-side, not tiled - layers_path = os.path.splitext(archive_path)[0] + ".vectors_and_internal_cogs.geojson" + layers_path = ( + os.path.splitext(archive_path)[0] + ".vectors_and_internal_cogs.geojson" + ) with open(layers_path, "w") as f: json.dump({"type": "FeatureCollection", "features": layer_features}, f) layers_size = os.path.getsize(layers_path) diff --git a/geoapi/tasks/published_ds_maps.py b/geoapi/tasks/published_ds_maps.py index 30633d6c..6c1a942a 100644 --- a/geoapi/tasks/published_ds_maps.py +++ b/geoapi/tasks/published_ds_maps.py @@ -223,8 +223,8 @@ def generate_published_ds_maps_pmtiles(): settings.APP_ENV, ) return - features, layer_features, stats = ( - PublishedMapsExportService.build_features(session, published_maps) + features, layer_features, stats = PublishedMapsExportService.build_features( + session, published_maps ) if not features: diff --git a/geoapi/tests/custom/designsafe/test_published_export.py b/geoapi/tests/custom/designsafe/test_published_export.py index 64660c5f..7d00646d 100644 --- a/geoapi/tests/custom/designsafe/test_published_export.py +++ b/geoapi/tests/custom/designsafe/test_published_export.py @@ -129,8 +129,13 @@ def test_build_features_tiles_markers_and_lists_layer_footprints(db_session): assert "designsafe.storage.published/PRJ-1" in props["ds_project_url"] assert props["hazmapper_url"].endswith(f"/project-public/{project.uuid}") assert "selectedFeature" not in props["hazmapper_url"] - for gone in ("project_uuid", "ds_project_id", "ds_doi", "has_assets", - "created_date"): + for gone in ( + "project_uuid", + "ds_project_id", + "ds_doi", + "has_assets", + "created_date", + ): assert gone not in props assert by_type["point"][0]["properties"]["feature_id"] == point.id From cae117c228990b97687136cb41cf397d23ebaf64 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 31 Jul 2026 14:42:25 -0500 Subject: [PATCH 21/23] Skip large cogs/rectangles from PRJ-6361 --- geoapi/custom/designsafe/published_export.py | 32 +++++++++++++++++-- .../designsafe/test_published_export.py | 1 + 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/geoapi/custom/designsafe/published_export.py b/geoapi/custom/designsafe/published_export.py index 89e0d09a..801103be 100644 --- a/geoapi/custom/designsafe/published_export.py +++ b/geoapi/custom/designsafe/published_export.py @@ -21,6 +21,15 @@ # A footprint wider/taller than this many degrees is logged as a likely georef error. LARGE_EXTENT_WARN_DEG = 30 +# TODO(from WG-703): brittle, project-specific guard. PRJ-6361 holds nation-wide image +# footprints (a large filled rectangle per image) that render poorly on the +# consumer (e.g. Recon Portal), so we leave them out for now. When the next such +# case appears, replace this with a general policy or figure out how to display +# these large datasets better on the consumer. Skip any PRJ-6361 geometry wider +# or taller than this many degrees. +PRJ_6361_PROJECT_ID = "PRJ-6361" +PRJ_6361_MAX_EXTENT_DEG = 4 + # asset_type (FeatureAsset) -> feature_type for tiled features. "vector" is handled # separately as a layer footprint; other unlisted types fall through to a # geometry-based point/shape classification. @@ -53,7 +62,8 @@ def build_features( ``layer_features`` are footprints for internal COGs and PMTiles-vector uploads, written to the companion GeoJSON and rendered client-side. ``stats`` has ``feature_count`` (tiled), ``cog_count``, - ``vector_count``, ``project_count`` and ``total`` (tiled). + ``vector_count``, ``skipped_count`` (oversized geometries dropped), + ``project_count`` and ``total`` (tiled). """ geoapi_base = get_deployed_geoapi_url() hazmapper_base = get_deployed_hazmapper_url() @@ -63,6 +73,7 @@ def build_features( feature_count = 0 cog_count = 0 vector_count = 0 + skipped_count = 0 for published in published_maps: project = published.project @@ -105,6 +116,21 @@ def build_features( vector_count += 1 continue + if published.designsafe_project_id == PRJ_6361_PROJECT_ID: + minx, miny, maxx, maxy = shapely.geometry.shape(geometry).bounds + width_deg, height_deg = abs(maxx - minx), abs(maxy - miny) + if max(width_deg, height_deg) > PRJ_6361_MAX_EXTENT_DEG: + logger.info( + "Skipping feature %s: geometry spans %.1f x %.1f deg; " + "a feature this large renders poorly on the consumer " + "(e.g. Recon Portal), so leaving out for now.", + feature.id, + width_deg, + height_deg, + ) + skipped_count += 1 + continue + properties = dict(common) properties.update( { @@ -133,16 +159,18 @@ def build_features( "feature_count": feature_count, "cog_count": cog_count, "vector_count": vector_count, + "skipped_count": skipped_count, "project_count": len(published_maps), "total": len(features), } logger.info( "Public export built %s tiled feature(s) + %s COG + %s vector " - "footprint(s) across %s project(s).", + "footprint(s) across %s project(s); skipped %s oversized geometry(ies).", feature_count, cog_count, vector_count, stats["project_count"], + skipped_count, ) return features, layer_features, stats diff --git a/geoapi/tests/custom/designsafe/test_published_export.py b/geoapi/tests/custom/designsafe/test_published_export.py index 7d00646d..1c69b877 100644 --- a/geoapi/tests/custom/designsafe/test_published_export.py +++ b/geoapi/tests/custom/designsafe/test_published_export.py @@ -107,6 +107,7 @@ def test_build_features_tiles_markers_and_lists_layer_footprints(db_session): "feature_count": 2, "cog_count": 1, "vector_count": 1, + "skipped_count": 0, "project_count": 1, "total": 2, } From a919dc9e9e6ef293121ce9545e783b9998f98399 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 31 Jul 2026 18:02:34 -0500 Subject: [PATCH 22/23] Fix skipping --- geoapi/custom/designsafe/published_export.py | 53 +++++++++++++------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/geoapi/custom/designsafe/published_export.py b/geoapi/custom/designsafe/published_export.py index 801103be..769e847b 100644 --- a/geoapi/custom/designsafe/published_export.py +++ b/geoapi/custom/designsafe/published_export.py @@ -21,12 +21,12 @@ # A footprint wider/taller than this many degrees is logged as a likely georef error. LARGE_EXTENT_WARN_DEG = 30 -# TODO(from WG-703): brittle, project-specific guard. PRJ-6361 holds nation-wide image -# footprints (a large filled rectangle per image) that render poorly on the +# TODO(from WG-703): brittle, project-specific guard. PRJ-6361 holds nation-wide +# layers (large image footprints and a worldwide COG) that render poorly on the # consumer (e.g. Recon Portal), so we leave them out for now. When the next such # case appears, replace this with a general policy or figure out how to display -# these large datasets better on the consumer. Skip any PRJ-6361 geometry wider -# or taller than this many degrees. +# these large datasets better on the consumer. Skip any PRJ-6361 tiled feature or +# COG footprint wider or taller than this many degrees. PRJ_6361_PROJECT_ID = "PRJ-6361" PRJ_6361_MAX_EXTENT_DEG = 4 @@ -62,8 +62,8 @@ def build_features( ``layer_features`` are footprints for internal COGs and PMTiles-vector uploads, written to the companion GeoJSON and rendered client-side. ``stats`` has ``feature_count`` (tiled), ``cog_count``, - ``vector_count``, ``skipped_count`` (oversized geometries dropped), - ``project_count`` and ``total`` (tiled). + ``vector_count``, ``skipped_count`` (oversized tiled features / COG + footprints dropped), ``project_count`` and ``total`` (tiled). """ geoapi_base = get_deployed_geoapi_url() hazmapper_base = get_deployed_hazmapper_url() @@ -118,16 +118,9 @@ def build_features( if published.designsafe_project_id == PRJ_6361_PROJECT_ID: minx, miny, maxx, maxy = shapely.geometry.shape(geometry).bounds - width_deg, height_deg = abs(maxx - minx), abs(maxy - miny) - if max(width_deg, height_deg) > PRJ_6361_MAX_EXTENT_DEG: - logger.info( - "Skipping feature %s: geometry spans %.1f x %.1f deg; " - "a feature this large renders poorly on the consumer " - "(e.g. Recon Portal), so leaving out for now.", - feature.id, - width_deg, - height_deg, - ) + if cls._skip_prj6361_oversized( + "feature", feature.id, abs(maxx - minx), abs(maxy - miny) + ): skipped_count += 1 continue @@ -151,9 +144,17 @@ def build_features( .all() ): footprint = cls._cog_footprint(tile_server, common, geoapi_base) - if footprint: - layer_features.append(footprint) - cog_count += 1 + if not footprint: + continue + if published.designsafe_project_id == PRJ_6361_PROJECT_ID: + west, south, east, north = footprint["properties"]["bounds"] + if cls._skip_prj6361_oversized( + "COG", tile_server.id, abs(east - west), abs(north - south) + ): + skipped_count += 1 + continue + layer_features.append(footprint) + cog_count += 1 stats = { "feature_count": feature_count, @@ -187,6 +188,20 @@ def _feature_type(feature) -> str: def _hazmapper_url(hazmapper_base, project_uuid) -> str: return f"{hazmapper_base}/project-public/{project_uuid}" + @staticmethod + def _skip_prj6361_oversized(label, obj_id, width_deg, height_deg) -> bool: + if max(width_deg, height_deg) <= PRJ_6361_MAX_EXTENT_DEG: + return False + logger.info( + "Skipping %s %s: extent spans %.1f x %.1f deg; too large to display " + "well on the consumer (e.g. Recon Portal), so leaving out for now.", + label, + obj_id, + width_deg, + height_deg, + ) + return True + @classmethod def _cog_footprint(cls, tile_server, common, geoapi_base) -> Optional[dict]: tile_options = tile_server.tileOptions or {} From 86494f45041068a188fa11bd398298896fdc93d7 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 31 Jul 2026 22:11:19 -0500 Subject: [PATCH 23/23] add /assets/public location --- devops/local_conf/nginx.conf | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/devops/local_conf/nginx.conf b/devops/local_conf/nginx.conf index 45632bac..1a5ec653 100644 --- a/devops/local_conf/nginx.conf +++ b/devops/local_conf/nginx.conf @@ -17,6 +17,15 @@ http { ~*file:///assets/([0-9]+)/ $1; # e.g. ?url=file:///assets/3/... default ""; } + + # Cache-Control for the public area: the versioned .pmtiles archives and their + # companion layer .geojson are immutable so can be cached hard; manifest.json + # changes nightly so it revalidates. + map $uri $public_asset_cache_control { + ~*\.pmtiles$ "public, max-age=31536000, immutable"; + ~*\.geojson$ "public, max-age=31536000, immutable"; + default "no-cache"; + } server { include /etc/nginx/mime.types; client_max_body_size 1g; @@ -54,6 +63,41 @@ http { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } + # Assets available to everyone -- currently only the PMTiles archive of + # all published DesignSafe projects (WG-703). A separate block from + # `location /assets` so it skips that block's auth_request. + location /assets/public/ { + alias /assets/public/; + + # .pmtiles/.geojson aren't in the default mime.types; the consumer keys off them + types { + application/vnd.pmtiles pmtiles; + application/geo+json geojson; + application/json json; + } + default_type application/octet-stream; + + gzip off; + add_header Accept-Ranges bytes always; + + add_header Cache-Control $public_asset_cache_control always; + + add_header Access-Control-Allow-Origin * always; + add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always; + add_header Access-Control-Allow-Headers "Range" always; + add_header Access-Control-Expose-Headers + "Content-Length, Content-Range, Accept-Ranges" always; + + if ($request_method = OPTIONS) { + add_header Access-Control-Allow-Origin * always; + add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always; + add_header Access-Control-Allow-Headers "Range" always; + add_header Access-Control-Max-Age 86400 always; + add_header Content-Length 0 always; + return 204; + } + } + location /assets { # Call geoapi to check access before serving file auth_request /auth-check;