diff --git a/devops/local_conf/nginx.conf b/devops/local_conf/nginx.conf index 45632bac..1a5ec653 100644 --- a/devops/local_conf/nginx.conf +++ b/devops/local_conf/nginx.conf @@ -17,6 +17,15 @@ http { ~*file:///assets/([0-9]+)/ $1; # e.g. ?url=file:///assets/3/... default ""; } + + # Cache-Control for the public area: the versioned .pmtiles archives and their + # companion layer .geojson are immutable so can be cached hard; manifest.json + # changes nightly so it revalidates. + map $uri $public_asset_cache_control { + ~*\.pmtiles$ "public, max-age=31536000, immutable"; + ~*\.geojson$ "public, max-age=31536000, immutable"; + default "no-cache"; + } server { include /etc/nginx/mime.types; client_max_body_size 1g; @@ -54,6 +63,41 @@ http { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } + # Assets available to everyone -- currently only the PMTiles archive of + # all published DesignSafe projects (WG-703). A separate block from + # `location /assets` so it skips that block's auth_request. + location /assets/public/ { + alias /assets/public/; + + # .pmtiles/.geojson aren't in the default mime.types; the consumer keys off them + types { + application/vnd.pmtiles pmtiles; + application/geo+json geojson; + application/json json; + } + default_type application/octet-stream; + + gzip off; + add_header Accept-Ranges bytes always; + + add_header Cache-Control $public_asset_cache_control always; + + add_header Access-Control-Allow-Origin * always; + add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always; + add_header Access-Control-Allow-Headers "Range" always; + add_header Access-Control-Expose-Headers + "Content-Length, Content-Range, Accept-Ranges" always; + + if ($request_method = OPTIONS) { + add_header Access-Control-Allow-Origin * always; + add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always; + add_header Access-Control-Allow-Headers "Range" always; + add_header Access-Control-Max-Age 86400 always; + add_header Content-Length 0 always; + return 204; + } + } + location /assets { # Call geoapi to check access before serving file auth_request /auth-check; diff --git a/geoapi/app.py b/geoapi/app.py index 5c55e56c..bc955eb8 100644 --- a/geoapi/app.py +++ b/geoapi/app.py @@ -30,6 +30,7 @@ from geoapi.models import User from geoapi.routes import api_router from geoapi.settings import settings +from geoapi.log import logger from geoapi.db import litestar_sqlalchemy_config, managed_litestar_db_session from geoapi.exceptions import ( InvalidGeoJSON, @@ -296,8 +297,26 @@ async def retrieve_session_user_handler( } +def _enqueue_published_ds_maps_cold_start(_app: "Litestar") -> None: + """On startup, enqueue an initial published-DS-maps build if none exists yet.""" + # tests exercise the full app lifespan via TestClient; don't auto-enqueue a + # real task (which would hit the live DesignSafe API) during the test run + if settings.APP_ENV == "testing": + return + from geoapi.tasks.published_ds_maps import enqueue_generation_if_missing + + try: + enqueue_generation_if_missing() + except Exception: + # never let a cold-start convenience break app startup + logger.exception( + "Cold-start enqueue of published-DS-maps generation failed; continuing." + ) + + app = Litestar( route_handlers=[api_router], + on_startup=[_enqueue_published_ds_maps_cold_start], middleware=[ logging_middleware_config.middleware, cookie_session_config.middleware, diff --git a/geoapi/celery_app.py b/geoapi/celery_app.py index 3348ab21..38a465ef 100644 --- a/geoapi/celery_app.py +++ b/geoapi/celery_app.py @@ -1,4 +1,5 @@ from celery import Celery +from celery.schedules import crontab from datetime import timedelta from geoapi.settings import settings @@ -26,6 +27,7 @@ "geoapi.tasks.projects", "geoapi.tasks.external_data", "geoapi.tasks.file_location_check", + "geoapi.tasks.published_ds_maps", ) # Define the queues @@ -58,4 +60,8 @@ "task": "geoapi.tasks.external_data.refresh_projects_watch_users", "schedule": timedelta(minutes=30), }, + "generate_published_ds_maps_pmtiles": { + "task": "geoapi.tasks.published_ds_maps.generate_published_ds_maps_pmtiles", + "schedule": crontab(hour=3, minute=0), + }, } diff --git a/geoapi/custom/designsafe/published_export.py b/geoapi/custom/designsafe/published_export.py new file mode 100644 index 00000000..769e847b --- /dev/null +++ b/geoapi/custom/designsafe/published_export.py @@ -0,0 +1,308 @@ +import shapely.geometry +from typing import List, Optional, Tuple +from urllib.parse import quote + +from geoapi.log import logging +from geoapi.models import Feature, TileServer +from geoapi.custom.designsafe.published_maps import PublishedMap +from geoapi.utils.client_backend import ( + get_deployed_geoapi_url, + get_deployed_hazmapper_url, +) + +logger = logging.getLogger(__name__) + +# DesignSafe published-data browser URL for a PRJ-#### project. +DESIGNSAFE_PUBLISHED_BROWSER_URL = ( + "https://www.designsafe-ci.org/data/browser/public/" + "designsafe.storage.published/{designsafe_project_id}" +) + +# A footprint wider/taller than this many degrees is logged as a likely georef error. +LARGE_EXTENT_WARN_DEG = 30 + +# TODO(from WG-703): brittle, project-specific guard. PRJ-6361 holds nation-wide +# layers (large image footprints and a worldwide COG) that render poorly on the +# consumer (e.g. Recon Portal), so we leave them out for now. When the next such +# case appears, replace this with a general policy or figure out how to display +# these large datasets better on the consumer. Skip any PRJ-6361 tiled feature or +# COG footprint wider or taller than this many degrees. +PRJ_6361_PROJECT_ID = "PRJ-6361" +PRJ_6361_MAX_EXTENT_DEG = 4 + +# asset_type (FeatureAsset) -> feature_type for tiled features. "vector" is handled +# separately as a layer footprint; other unlisted types fall through to a +# geometry-based point/shape classification. +_ASSET_TYPE_TO_FEATURE_TYPE = { + "image": "image", + "video": "video", + "point_cloud": "point_cloud", + "streetview": "streetview", + "questionnaire": "questionnaire", +} + + +class PublishedMapsExportService: + """Turn selected published maps into GeoJSON for the public archive. + + Emits per-Hazmapper-feature dicts for the tiled PMTiles archive, plus layer + footprints (internal COGs and PMTiles-vector uploads) for a companion GeoJSON + rendered client-side. Every feature carries a property set that Recon Portal + uses. + """ + + @classmethod + def build_features( + cls, database_session, published_maps: List[PublishedMap] + ) -> Tuple[List[dict], List[dict], dict]: + """Build the tiled features and the layer footprints. + + :return: ``(features, layer_features, stats)``. ``features`` are the + per-Hazmapper-feature dicts tiled into the PMTiles archive. + ``layer_features`` are footprints for internal COGs and PMTiles-vector + uploads, written to the companion GeoJSON and rendered client-side. + ``stats`` has ``feature_count`` (tiled), ``cog_count``, + ``vector_count``, ``skipped_count`` (oversized tiled features / COG + footprints dropped), ``project_count`` and ``total`` (tiled). + """ + geoapi_base = get_deployed_geoapi_url() + hazmapper_base = get_deployed_hazmapper_url() + + features: List[dict] = [] + layer_features: List[dict] = [] + feature_count = 0 + cog_count = 0 + vector_count = 0 + skipped_count = 0 + + for published in published_maps: + project = published.project + common = { + "hazmapper_project_uuid": str(project.uuid), + "hazmapper_project_name": project.name, + "ds_project_name": published.designsafe_project_title, + "ds_project_url": DESIGNSAFE_PUBLISHED_BROWSER_URL.format( + designsafe_project_id=published.designsafe_project_id + ), + "hazmapper_url": cls._hazmapper_url(hazmapper_base, project.uuid), + } + + for feature in ( + database_session.query(Feature) + .filter(Feature.project_id == project.id) + .all() + ): + try: + geometry = feature.geometry + except Exception: + logger.warning( + "Feature %s has unreadable geometry; skipping.", feature.id + ) + continue + if not geometry: + continue + + # a PMTiles-vector upload -> layer footprint (pointer to its own + # tiles), not tiled here + vector_asset = next( + (a for a in feature.assets if a.asset_type == "vector"), None + ) + if vector_asset: + layer_features.append( + cls._vector_footprint( + feature, geometry, vector_asset, common, geoapi_base + ) + ) + vector_count += 1 + continue + + if published.designsafe_project_id == PRJ_6361_PROJECT_ID: + minx, miny, maxx, maxy = shapely.geometry.shape(geometry).bounds + if cls._skip_prj6361_oversized( + "feature", feature.id, abs(maxx - minx), abs(maxy - miny) + ): + skipped_count += 1 + continue + + properties = dict(common) + properties.update( + { + "feature_id": feature.id, + "feature_type": cls._feature_type(feature), + } + ) + features.append( + {"type": "Feature", "geometry": geometry, "properties": properties} + ) + feature_count += 1 + + for tile_server in ( + database_session.query(TileServer) + .filter(TileServer.project_id == project.id) + .filter(TileServer.internal.is_(True)) + .filter(TileServer.kind == "cog") + .all() + ): + footprint = cls._cog_footprint(tile_server, common, geoapi_base) + if not footprint: + continue + if published.designsafe_project_id == PRJ_6361_PROJECT_ID: + west, south, east, north = footprint["properties"]["bounds"] + if cls._skip_prj6361_oversized( + "COG", tile_server.id, abs(east - west), abs(north - south) + ): + skipped_count += 1 + continue + layer_features.append(footprint) + cog_count += 1 + + stats = { + "feature_count": feature_count, + "cog_count": cog_count, + "vector_count": vector_count, + "skipped_count": skipped_count, + "project_count": len(published_maps), + "total": len(features), + } + logger.info( + "Public export built %s tiled feature(s) + %s COG + %s vector " + "footprint(s) across %s project(s); skipped %s oversized geometry(ies).", + feature_count, + cog_count, + vector_count, + stats["project_count"], + skipped_count, + ) + return features, layer_features, stats + + @staticmethod + def _feature_type(feature) -> str: + for asset in feature.assets: + mapped = _ASSET_TYPE_TO_FEATURE_TYPE.get(asset.asset_type) + if mapped: + return mapped + geometry_type = ((feature.geometry or {}).get("type") or "").lower() + return "point" if "point" in geometry_type else "shape" + + @staticmethod + def _hazmapper_url(hazmapper_base, project_uuid) -> str: + return f"{hazmapper_base}/project-public/{project_uuid}" + + @staticmethod + def _skip_prj6361_oversized(label, obj_id, width_deg, height_deg) -> bool: + if max(width_deg, height_deg) <= PRJ_6361_MAX_EXTENT_DEG: + return False + logger.info( + "Skipping %s %s: extent spans %.1f x %.1f deg; too large to display " + "well on the consumer (e.g. Recon Portal), so leaving out for now.", + label, + obj_id, + width_deg, + height_deg, + ) + return True + + @classmethod + def _cog_footprint(cls, tile_server, common, geoapi_base) -> Optional[dict]: + tile_options = tile_server.tileOptions or {} + bounds = tile_options.get("bounds") + # bounds is leaflet-style [[minLat, minLng], [maxLat, maxLng]] + if not bounds or len(bounds) != 2: + logger.warning( + "Internal COG tile_server %s has no usable bounds; skipping its " + "footprint.", + tile_server.id, + ) + return None + (south, west), (north, east) = bounds + ring = [ + [west, south], + [east, south], + [east, north], + [west, north], + [west, south], + ] + geometry = {"type": "Polygon", "coordinates": [ring]} + cls._warn_if_large("COG", tile_server.id, abs(east - west), abs(north - south)) + + # TileServerLayer descriptor the consumer feeds to its layer builder. + descriptor = { + "id": tile_server.id, + "name": tile_server.name, + "type": tile_server.type, + "kind": tile_server.kind, + "internal": tile_server.internal, + "uuid": str(tile_server.uuid) if tile_server.uuid else None, + "url": tile_server.url, + "attribution": tile_server.attribution, + "tileOptions": tile_options, + "uiOptions": tile_server.uiOptions or {}, + } + + properties = dict(common) + properties.update( + { + "feature_id": None, + "feature_type": "cog", + "cog_url": cls._build_cog_url(tile_server, geoapi_base), + "bounds": [west, south, east, north], + "tile_layer": descriptor, + } + ) + return {"type": "Feature", "geometry": geometry, "properties": properties} + + @classmethod + def _vector_footprint(cls, feature, geometry, vector_asset, common, geoapi_base): + """Layer footprint for a PMTiles-vector upload: the feature bbox + a pointer + to the upload's own PMTiles archive.""" + minx, miny, maxx, maxy = shapely.geometry.shape(geometry).bounds + cls._warn_if_large("Vector", feature.id, abs(maxx - minx), abs(maxy - miny)) + properties = dict(common) + properties.update( + { + "feature_id": feature.id, + "feature_type": "vector", + "pmtiles_url": f"{geoapi_base}/assets/{vector_asset.path}", + "bounds": [minx, miny, maxx, maxy], + } + ) + return {"type": "Feature", "geometry": geometry, "properties": properties} + + @staticmethod + def _warn_if_large(kind, obj_id, width_deg, height_deg): + if max(width_deg, height_deg) > LARGE_EXTENT_WARN_DEG: + logger.warning( + "%s %s footprint extent %.2f x %.2f deg is unusually large " + "(possible georef error); still included.", + kind, + obj_id, + width_deg, + height_deg, + ) + else: + logger.info( + "%s %s footprint extent %.3f x %.3f deg.", + kind, + obj_id, + width_deg, + height_deg, + ) + + @staticmethod + def _build_cog_url(tile_server, geoapi_base) -> str: + """Pre-resolved TiTiler tile URL template for an internal COG. + + Mirrors the hazmapper frontend's resolveTileUrl: wrap the internal file + path as ``file://...`` (URL-encoded) and append any render options. The + ``{z}/{x}/{y}`` placeholders are left literal for the consumer. + """ + file_url = "file://" + tile_server.url + encoded = quote(file_url, safe="") + url = ( + f"{geoapi_base}/tiles/cog/tiles/WebMercatorQuad/" + f"{{z}}/{{x}}/{{y}}.png?url={encoded}" + ) + render_options = (tile_server.uiOptions or {}).get("renderOptions") or {} + for key, value in render_options.items(): + url += f"&{key}={quote(str(value), safe='')}" + return url diff --git a/geoapi/custom/designsafe/published_maps.py b/geoapi/custom/designsafe/published_maps.py new file mode 100644 index 00000000..463f8309 --- /dev/null +++ b/geoapi/custom/designsafe/published_maps.py @@ -0,0 +1,193 @@ +import time +from dataclasses import dataclass +from typing import Iterator, List, Optional + +import requests + +from geoapi.log import logging +from geoapi.models import Project +from geoapi.settings import settings + +logger = logging.getLogger(__name__) + +# DesignSafe publications API (public, unauthenticated). This is intentionally a +# module-level constant rather than a per-environment setting: "published" is a +# production DesignSafe curation act, and the maps embedded in a publication are +# tagged with the deployment that registered them (see below), so pointing at a +# single publications host and filtering by deployment is what keeps each geoapi +# tier scoped to its own maps. Overridable in tests. +DESIGNSAFE_PUBLICATIONS_API = "https://www.designsafe-ci.org/api/publications/v2" + +# Page size when walking the publications list. +PUBLICATIONS_PAGE_LIMIT = 100 + +# HTTP timeout for a single DesignSafe request. +REQUEST_TIMEOUT_SECONDS = 30 + + +@dataclass +class PublishedMap: + """A Hazmapper map that belongs to a published DesignSafe project *and* to + this geoapi deployment. + + ``project`` is the local geoapi Project (already confirmed public); the + remaining fields carry the DesignSafe publication context needed to enrich + exported features. + """ + + project: Project + map_uuid: str + map_name: Optional[str] + map_path: Optional[str] + designsafe_project_id: str # PRJ-#### + designsafe_project_title: Optional[str] + designsafe_doi: Optional[str] + + +class PublishedMapsService: + """Select the Hazmapper maps that back public, published DesignSafe projects + for the current deployment. + + The traversal mirrors the qgis plugin's discovery script + (``hazmapper-qgis-plugin/scripts/designsafe_hazmapper_discovery.py``): start + from the DesignSafe publications API, and for each publication read the + ``baseProject.hazmapperMaps`` it embeds. Each embedded map is tagged with the + ``deployment`` (the ``APP_ENV`` of the geoapi that registered it -- see + ``custom/designsafe/project.py``), which is how a single publication can + reference maps on more than one tier. + + A map is selected only when all three conditions hold: + 1. published -- it appears in a DesignSafe publication + 2. this tier -- ``deployment == settings.APP_ENV`` + 3. public -- the local Project joined by ``uuid`` has ``public = True`` + + A map that matches this deployment but whose local project is missing or not + public is logged and skipped, not raised -- one bad map must not sink the + nightly run. + """ + + # HTTP entry points are isolated here so tests can patch them without a + # network round-trip. + publications_api = DESIGNSAFE_PUBLICATIONS_API + + # Politeness delay between per-project detail requests. Set to 0 in tests. + inter_request_sleep_seconds = 0.1 + + @classmethod + def _get_json(cls, url: str) -> dict: + response = requests.get(url, timeout=REQUEST_TIMEOUT_SECONDS) + response.raise_for_status() + return response.json() + + @classmethod + def _iter_published_projects(cls) -> Iterator[dict]: + """Yield every published project (paged) from the publications list.""" + offset = 0 + while True: + data = cls._get_json( + f"{cls.publications_api}?offset={offset}&limit={PUBLICATIONS_PAGE_LIMIT}" + ) + projects = data.get("result", []) + if not projects: + return + for project in projects: + yield project + # a short page means we've reached the end + if len(projects) < PUBLICATIONS_PAGE_LIMIT: + return + offset += PUBLICATIONS_PAGE_LIMIT + + @classmethod + def _fetch_base_project(cls, designsafe_project_id: str) -> dict: + """Return the ``baseProject`` block for a single publication. + + Carries both the embedded ``hazmapperMaps`` and the publication ``dois``. + """ + detail = cls._get_json(f"{cls.publications_api}/{designsafe_project_id}") + return detail.get("baseProject") or {} + + @classmethod + def get_published_maps(cls, database_session) -> List[PublishedMap]: + """Resolve the published, this-deployment, public maps to local Projects. + + :param database_session: SQLAlchemy session bound to this deployment's DB + :return: selected maps (may be empty -- e.g. a tier with no published maps) + """ + deployment = settings.APP_ENV + selected: List[PublishedMap] = [] + seen_uuids: set[str] = set() + deployment_matched = 0 + + for project in cls._iter_published_projects(): + designsafe_project_id = project.get("projectId") + if not designsafe_project_id: + continue + designsafe_title = project.get("title") + + base_project = cls._fetch_base_project(designsafe_project_id) + dois = base_project.get("dois") or [] + designsafe_doi = dois[0] if dois else None + + for hazmapper_map in base_project.get("hazmapperMaps") or []: + map_uuid = hazmapper_map.get("uuid") + if not map_uuid: + continue + + # Condition 2: only maps registered by *this* geoapi deployment. A map + # tagged for another tier lives in another tier's DB; skip quietly. + if hazmapper_map.get("deployment") != deployment: + continue + + deployment_matched += 1 + if map_uuid in seen_uuids: + continue + + # Condition 3: the map must resolve to a local, public project. + local_project = ( + database_session.query(Project) + .filter(Project.uuid == map_uuid) + .one_or_none() + ) + if local_project is None: + logger.warning( + "Published map tagged for this deployment label is not in " + "this database; skipping. This is expected when the label is " + "shared across instances (e.g. another 'local'/'dev'). " + f"deployment:{deployment} designsafe_project:{designsafe_project_id} " + f"map_uuid:{map_uuid}" + ) + continue + if not local_project.public: + # Published on DesignSafe but toggled non-public in Hazmapper. + # Log at error level (an inconsistency worth noticing) but do + # not raise -- just exclude this one map. + logger.error( + "Published DesignSafe map resolves to a non-public project; " + "excluding it from the public archive. " + f"deployment:{deployment} designsafe_project:{designsafe_project_id} " + f"map_uuid:{map_uuid} project_id:{local_project.id}" + ) + continue + + seen_uuids.add(map_uuid) + selected.append( + PublishedMap( + project=local_project, + map_uuid=map_uuid, + map_name=hazmapper_map.get("name"), + map_path=hazmapper_map.get("path"), + designsafe_project_id=designsafe_project_id, + designsafe_project_title=designsafe_title, + designsafe_doi=designsafe_doi, + ) + ) + + if cls.inter_request_sleep_seconds: + time.sleep(cls.inter_request_sleep_seconds) + + logger.info( + f"Selected {len(selected)} published, public map(s) for deployment " + f"'{deployment}' (out of {deployment_matched} deployment-matched " + "published map(s))." + ) + return selected diff --git a/geoapi/misc/README.md b/geoapi/misc/README.md new file mode 100644 index 00000000..488508f1 --- /dev/null +++ b/geoapi/misc/README.md @@ -0,0 +1,25 @@ +# `misc/` — one-off operational scripts + +Ad-hoc scripts run by hand (not part of the API or the Celery schedule). + +## `dry_run_published_ds_maps.py` + +A dry run creating the published-DesignSafe-maps archive. + +It writes the archive to `/assets/tmp/published_ds_maps.pmtiles` + +### Run against the local dev stack + +```bash +docker exec -it geoapi_workers python misc/dry_run_published_ds_maps.py +``` + +### Run against production + +```bash +docker run --rm \ + --env-file /opt/portal/conf/secrets.env \ + -v /assets:/assets \ + taccwma/geoapi-workers: \ + python misc/dry_run_published_ds_maps.py +``` diff --git a/geoapi/misc/dry_run_published_ds_maps.py b/geoapi/misc/dry_run_published_ds_maps.py new file mode 100644 index 00000000..7beac929 --- /dev/null +++ b/geoapi/misc/dry_run_published_ds_maps.py @@ -0,0 +1,170 @@ +""" +Non-destructive dry run of creating published DesignSafe maps archive. Useful for testing out +on prod and staging systems before deployment. See README.md +""" + +import argparse +import json +import os +import shutil +import subprocess +import tempfile +import time +from collections import Counter, defaultdict + +from geoapi.custom.designsafe.published_export import PublishedMapsExportService +from geoapi.custom.designsafe.published_maps import PublishedMapsService +from geoapi.db import create_task_session +from geoapi.services.tippecanoe import ( + PUBLISHED_DS_MAPS_BASE_ZOOM, + PUBLISHED_DS_MAPS_MAX_ZOOM, + PUBLISHED_DS_MAPS_MIN_ZOOM, + TippecanoeService, +) +from geoapi.settings import settings +from geoapi.tasks.published_ds_maps import _bounds, _write_layer_files +from geoapi.utils.assets import get_temp_dir +from geoapi.utils.client_backend import ( + get_deployed_geoapi_url, + get_deployed_hazmapper_url, +) + + +def _human_bytes(n): + for unit in ("B", "KB", "MB", "GB"): + if n < 1024 or unit == "GB": + return f"{n:.1f} {unit}" + n /= 1024 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--out", + default=None, + help="output .pmtiles path (default: " + "/assets/tmp/published_ds_maps.pmtiles; never the real assets/public area)", + ) + args = parser.parse_args() + + print("=" * 72) + print("WG-703 published DesignSafe maps archive -- DRY RUN (non-destructive)") + print("=" * 72) + print(f"APP_ENV : {settings.APP_ENV}") + print(f"DB host / name : {settings.DB_HOST} / {settings.DB_NAME}") + print(f"geoapi base URL : {get_deployed_geoapi_url()}") + print(f"hazmapper base URL : {get_deployed_hazmapper_url()}") + print( + f"zoom range : z{PUBLISHED_DS_MAPS_MIN_ZOOM}-z{PUBLISHED_DS_MAPS_MAX_ZOOM} " + f"(all features kept at base z{PUBLISHED_DS_MAPS_BASE_ZOOM}+)" + ) + print("-" * 72) + + with create_task_session() as session: + published = PublishedMapsService.get_published_maps(session) + print(f"Published, public maps for this deployment: {len(published)}") + if not published: + print( + "\nNothing to measure. If you expected results, check that APP_ENV " + "matches the maps' deployment tag and that the DB is the right tier." + ) + return + features, layer_features, stats = PublishedMapsExportService.build_features( + session, published + ) + + print(f"Tiled features : {stats['feature_count']}") + print( + f"Layer footprints : {stats['cog_count']} COG + {stats['vector_count']} " + "vector (companion geojson)" + ) + + # per-layer (feature_type) breakdown + by_type = Counter(f["properties"].get("feature_type") for f in features) + print("\nPer-layer (feature_type) counts:") + for feature_type, count in by_type.most_common(): + print(f" {feature_type:<14} {count}") + + # per-feature property payload analysis (informs the projects.json decision) + total_props_bytes = 0 + per_key_bytes = defaultdict(int) + for f in features: + props = f["properties"] + total_props_bytes += len(json.dumps(props)) + for key, value in props.items(): + per_key_bytes[key] += len(json.dumps({key: value})) + avg_props = total_props_bytes / len(features) if features else 0 + print( + f"\nProperty payload : {_human_bytes(total_props_bytes)} total, " + f"{avg_props:.0f} bytes/feature avg (pre-tiling, uncompressed)" + ) + print("Heaviest property keys (total bytes across all features):") + for key, nbytes in sorted(per_key_bytes.items(), key=lambda kv: -kv[1])[:8]: + print(f" {key:<16} {_human_bytes(nbytes)}") + + bounds = _bounds(features + layer_features) + print(f"\nBounds [minx,miny,maxx,maxy]: {bounds}") + + # tile into a scratch path (NOT the real public area). Layer footprints (COGs + + # vectors) are not tiled -- they go to a companion geojson (see below). + archive_path = args.out or os.path.join( + str(get_temp_dir()), "published_ds_maps.pmtiles" + ) + os.makedirs(os.path.dirname(archive_path), exist_ok=True) + work_dir = tempfile.mkdtemp(prefix="layers_", dir=os.path.dirname(archive_path)) + written_total = 0 + try: + layer_files = _write_layer_files(features, work_dir) + print("\nPer-layer tiling (feature count / time / size):") + per_layer_pmtiles = [] + for layer_name, geojson_path in layer_files: + layer_pmtiles = os.path.join(work_dir, f"{layer_name}.pmtiles") + start = time.time() + count = TippecanoeService.geojson_layers_to_pmtiles( + [(layer_name, geojson_path)], layer_pmtiles + ) + elapsed = time.time() - start + written_total += count or 0 + per_layer_pmtiles.append(layer_pmtiles) + print( + f" {layer_name:<14} {count or 0:>7} feat {elapsed:>8.1f}s " + f"{_human_bytes(os.path.getsize(layer_pmtiles))}" + ) + + print(f"\nCombining {len(per_layer_pmtiles)} layer(s) -> {archive_path} ...") + subprocess.run( + ["tile-join", "--force", "--no-tile-size-limit", "-o", archive_path] + + per_layer_pmtiles, + check=True, + ) + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + # companion layer footprints (COGs + vectors), rendered client-side, not tiled + layers_path = ( + os.path.splitext(archive_path)[0] + ".vectors_and_internal_cogs.geojson" + ) + with open(layers_path, "w") as f: + json.dump({"type": "FeatureCollection", "features": layer_features}, f) + layers_size = os.path.getsize(layers_path) + + archive_size = os.path.getsize(archive_path) + print("-" * 72) + print(f"features tiled : {written_total} (source had {stats['feature_count']})") + if written_total != stats["feature_count"]: + print(" WARNING: count mismatch -- features were dropped!") + print(f"ARCHIVE SIZE : {_human_bytes(archive_size)} ({archive_size} bytes)") + print( + f"layers geojson : {_human_bytes(layers_size)} " + f"({stats['cog_count']} COG + {stats['vector_count']} vector)" + ) + print(f"Archive path : {archive_path}") + print(f"Layers path : {layers_path}") + print( + f"\n(Delete {archive_path} and {layers_path} when done. Nothing was written " + "to the real assets/public area or manifest.)" + ) + + +if __name__ == "__main__": + main() diff --git a/geoapi/routes/__init__.py b/geoapi/routes/__init__.py index 9ca26c91..70a6ca32 100644 --- a/geoapi/routes/__init__.py +++ b/geoapi/routes/__init__.py @@ -8,6 +8,7 @@ from .auth import AuthController from .webhooks import TaskStatusWebhookController from .websockets import websocket_handler +from .published_ds_maps import PublishedDsMapsController api_router = Router( path="/", @@ -20,5 +21,6 @@ AuthController, TaskStatusWebhookController, websocket_handler, + PublishedDsMapsController, ], ) diff --git a/geoapi/routes/published_ds_maps.py b/geoapi/routes/published_ds_maps.py new file mode 100644 index 00000000..6ff5ec86 --- /dev/null +++ b/geoapi/routes/published_ds_maps.py @@ -0,0 +1,32 @@ +from litestar import Controller, get +from litestar.response import Response + +from geoapi.log import logging +from geoapi.tasks.published_ds_maps import read_manifest + +logger = logging.getLogger(__name__) + + +class PublishedDsMapsController(Controller): + """Unauthenticated discovery endpoint for the published DesignSafe maps archive. + + It just serves the sidecar ``manifest.json`` written by the nightly + task. It never streams the archive as nginx serves the .pmtiles directly). + """ + + path = "/designsafe" + tags = ["public"] + + @get("/published-maps/pmtiles") + async def get_published_maps_pmtiles(self) -> Response: + """Return the manifest for the current combined public PMTiles archive of + published DesignSafe project maps (url + metadata), or 404 if none has + been generated yet. + """ + manifest = read_manifest() + if manifest is None: + return Response( + content={"detail": "No published-maps archive is available yet."}, + status_code=404, + ) + return Response(content=manifest, status_code=200) diff --git a/geoapi/services/tippecanoe.py b/geoapi/services/tippecanoe.py index a9f75a59..690a0ea6 100644 --- a/geoapi/services/tippecanoe.py +++ b/geoapi/services/tippecanoe.py @@ -1,5 +1,7 @@ import os +import re import subprocess +from typing import List, Optional, Tuple from geoapi.log import logging @@ -8,6 +10,16 @@ # tippecanoe binary; overridable for environments where it is not on PATH TIPPECANOE_BIN = os.environ.get("TIPPECANOE_BIN", "tippecanoe") +# Zoom config for the combined published DesignSafe maps archive (WG-703). +# min_zoom z2 -- world overview (features are thinned below base_zoom; lets +# the layer show a sparse scatter when zoomed out) +# base_zoom z8 -- every feature is kept at z8 and every zoom above (matches the +# ReconPortal's selectedEventZoomToLevel); below z8 they are thinned +# max_zoom z14 -- ~0.5m coordinate precision +PUBLISHED_DS_MAPS_MIN_ZOOM = 2 +PUBLISHED_DS_MAPS_BASE_ZOOM = 8 +PUBLISHED_DS_MAPS_MAX_ZOOM = 14 + # Floor for tippecanoe's guessed maximum zoom (-zg). # # `-zg` picks a max zoom from how far apart / how detailed the features are. @@ -91,3 +103,89 @@ def geojson_to_pmtiles( if result.stderr: logger.debug("tippecanoe output: %s", result.stderr) return output_path + + @staticmethod + def geojson_layers_to_pmtiles( + layers: List[Tuple[str, str]], + output_path: str, + min_zoom: int = PUBLISHED_DS_MAPS_MIN_ZOOM, + base_zoom: int = PUBLISHED_DS_MAPS_BASE_ZOOM, + max_zoom: int = PUBLISHED_DS_MAPS_MAX_ZOOM, + ) -> Optional[int]: + """ + Tile per-layer GeoJSON files into one PMTiles archive. + + Used for the combined published DesignSafe maps archive (WG-703). Unlike + ``geojson_to_pmtiles`` (per-feature vector ingest, which guesses maxzoom), + this pins a fixed zoom range and keeps features from ``base_zoom`` up + (``-B``), where the ReconPortal actually looks (its selectedEventZoomToLevel), + through overzoom. Below ``base_zoom`` they thin into a sparser + overview. Tiles that would blow past the size cap shed their densest + features at that zoom (``--drop-densest-as-needed``); those reappear at + higher zoom where the tile fits (so no feature is lost from the archive). + + Each entry becomes its own vector-tile layer, so the consumer (i.e. ReconPortal) + can style and toggle by feature type. + + :param layers: list of ``(layer_name, geojson_path)`` + :param output_path: path where the .pmtiles archive will be written + :param min_zoom: minimum zoom (default z2) + :param base_zoom: keep all features at/above this zoom; thin below (default z8) + :param max_zoom: maximum zoom (default z14) + :return: the feature count tippecanoe reports writing (for the caller's + source-vs-archive count check), or ``None`` if it can't be parsed + :raises RuntimeError: if the tippecanoe binary is missing or exits non-zero + :raises ValueError: if ``layers`` is empty + """ + if not layers: + raise ValueError("geojson_layers_to_pmtiles requires at least one layer") + + cmd = [ + TIPPECANOE_BIN, + "-o", + output_path, + "--force", # overwrite output_path if it already exists + "-Z", + str(min_zoom), + "-z", + str(max_zoom), + "-B", + str(base_zoom), + "--drop-densest-as-needed", + "--no-feature-limit", + "--no-tiny-polygon-reduction", + ] + for layer_name, geojson_path in layers: + # `-L name:file` reads file into its own named vector-tile layer + cmd += ["-L", f"{layer_name}:{geojson_path}"] + + logger.info("Running tippecanoe (public archive): %s", " ".join(cmd)) + try: + result = subprocess.run(cmd, capture_output=True, text=True) + except FileNotFoundError as e: + raise RuntimeError( + f"tippecanoe binary not found (looked for '{TIPPECANOE_BIN}'). " + "Install tippecanoe or set the TIPPECANOE_BIN environment variable." + ) from e + + if result.returncode != 0: + logger.error( + "tippecanoe failed (exit %s): %s", result.returncode, result.stderr + ) + raise RuntimeError( + f"tippecanoe failed with exit code {result.returncode}: " + f"{result.stderr.strip()}" + ) + + return TippecanoeService._parse_written_feature_count(result.stderr) + + @staticmethod + def _parse_written_feature_count(stderr: str) -> Optional[int]: + """Pull the feature count out of tippecanoe's summary line. + + tippecanoe prints e.g. ``"3 features, 81 bytes of geometry ..."`` to + stderr; this is the number of distinct input features it kept (it does + not match the earlier ``"Read N million features"`` progress line). + """ + match = re.search(r"(\d+) features,", stderr or "") + return int(match.group(1)) if match else None diff --git a/geoapi/tasks/published_ds_maps.py b/geoapi/tasks/published_ds_maps.py new file mode 100644 index 00000000..6c1a942a --- /dev/null +++ b/geoapi/tasks/published_ds_maps.py @@ -0,0 +1,340 @@ +import json +import os +import shutil +import tempfile +import time +from datetime import datetime, timezone +from typing import List, Optional, Tuple + +import redis +import shapely.geometry + +from geoapi.celery_app import app +from geoapi.custom.designsafe.published_export import PublishedMapsExportService +from geoapi.custom.designsafe.published_maps import PublishedMapsService +from geoapi.db import create_task_session +from geoapi.log import logger +from geoapi.services.tippecanoe import ( + PUBLISHED_DS_MAPS_MAX_ZOOM, + PUBLISHED_DS_MAPS_MIN_ZOOM, + TippecanoeService, +) +from geoapi.settings import settings +from geoapi.utils.assets import get_temp_dir +from geoapi.utils.client_backend import get_deployed_geoapi_url + +# Main pmtiles archive that will contain all published ds map data in one file +# (COG and PMTiles-vector layers are the exception, see LAYERS_SUFFIX below) +ARCHIVE_PREFIX = "published_ds_maps_" +ARCHIVE_SUFFIX = ".pmtiles" +# Companion GeoJSON of layer footprints (internal COGs and PMTiles-vector uploads), +# rendered client-side rather than tiled. +LAYERS_SUFFIX = ".vectors_and_internal_cogs.geojson" + +# Bumped when the manifest/property schema changes in a way consumers must notice. +SCHEMA_VERSION = 1 + +# Prune archives older than this, but never the one the manifest references. 30h +# (not 24h) leaves margin for a late run so a predecessor isn't removed early; so +# we should be keeping two generations. +PRUNE_AGE_SECONDS = 30 * 3600 + +# Redis lock so a nightly beat, a startup enqueue, and a manual trigger can't run +# two tippecanoe builds at once. +GENERATION_LOCK_KEY = "published_ds_maps:generation" +GENERATION_LOCK_TIMEOUT_SECONDS = 2 * 3600 + + +def public_asset_dir() -> str: + return os.path.join(settings.ASSETS_BASE_DIR, "public") + + +def manifest_path() -> str: + return os.path.join(public_asset_dir(), "manifest.json") + + +def read_manifest() -> Optional[dict]: + """Return the current manifest dict, or None if there isn't one yet.""" + try: + with open(manifest_path()) as f: + return json.load(f) + except (FileNotFoundError, ValueError): + return None + + +def _public_asset_url(filename: str) -> str: + return f"{get_deployed_geoapi_url()}/assets/public/{filename}" + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _timestamp(now: datetime) -> str: + return now.strftime("%Y%m%dT%H%M%SZ") + + +def _archive_filename(now: datetime) -> str: + return f"{ARCHIVE_PREFIX}{_timestamp(now)}{ARCHIVE_SUFFIX}" + + +def _layers_filename(now: datetime) -> str: + return f"{ARCHIVE_PREFIX}{_timestamp(now)}{LAYERS_SUFFIX}" + + +def _write_layer_files(features: List[dict], work_dir: str) -> List[Tuple[str, str]]: + """Write features grouped by ``feature_type`` to newline-delimited GeoJSON. + + :return: list of ``(layer_name, file_path)`` for tippecanoe named layers + """ + grouped: dict = {} + for feature in features: + layer = feature["properties"].get("feature_type") or "feature" + grouped.setdefault(layer, []).append(feature) + + layer_files: List[Tuple[str, str]] = [] + for layer, layer_features in grouped.items(): + path = os.path.join(work_dir, f"{layer}.geojson") + with open(path, "w") as f: + for feature in layer_features: + f.write(json.dumps(feature)) + f.write("\n") + layer_files.append((layer, path)) + return layer_files + + +def _bounds(features: List[dict]) -> Optional[List[float]]: + """[minx, miny, maxx, maxy] over all feature geometries (lon/lat).""" + minx = miny = maxx = maxy = None + for feature in features: + geometry = feature.get("geometry") + if not geometry: + continue + fminx, fminy, fmaxx, fmaxy = shapely.geometry.shape(geometry).bounds + minx = fminx if minx is None else min(minx, fminx) + miny = fminy if miny is None else min(miny, fminy) + maxx = fmaxx if maxx is None else max(maxx, fmaxx) + maxy = fmaxy if maxy is None else max(maxy, fmaxy) + if minx is None: + return None + return [minx, miny, maxx, maxy] + + +def _verify_feature_count(source_count: int, written_count: Optional[int]) -> None: + """Guard against silent dropping (the main failure mode of tiling). + + Raises if tippecanoe wrote fewer features than we handed it, so the caller + aborts before overwriting a good archive/manifest. + """ + if written_count is None: + logger.warning( + "Could not parse tippecanoe's feature count; skipping the " + "source-vs-archive count check (expected %s).", + source_count, + ) + return + if written_count != source_count: + raise RuntimeError( + "Public archive feature count mismatch: handed tippecanoe " + f"{source_count} feature(s) but it wrote {written_count}. Features " + "were dropped -- aborting without touching the existing archive." + ) + logger.info("Public archive feature count verified: %s.", source_count) + + +def _prune_old_public_files(keep_filenames: set) -> None: + """Delete versioned archive/cogs files older than PRUNE_AGE_SECONDS, never the + just-written ones or the ones the manifest currently references.""" + directory = public_asset_dir() + manifest = read_manifest() or {} + keep = set(keep_filenames) | { + os.path.basename(manifest.get("url", "")), + os.path.basename(manifest.get("layers_url", "")), + } + now = time.time() + for name in os.listdir(directory): + if not name.startswith(ARCHIVE_PREFIX): + continue + if not (name.endswith(ARCHIVE_SUFFIX) or name.endswith(LAYERS_SUFFIX)): + continue + if name in keep: + continue + path = os.path.join(directory, name) + try: + age = now - os.path.getmtime(path) + except OSError: + continue + if age > PRUNE_AGE_SECONDS: + try: + os.remove(path) + logger.info( + "Pruned stale public file %s (age %.1fh).", name, age / 3600 + ) + except OSError: + logger.exception("Failed to prune public file %s.", name) + + +@app.task(queue="heavy") +def generate_published_ds_maps_pmtiles(): + """Nightly: build the combined public PMTiles archive for published DS maps. + + Selects this deployment's published, public maps, tiles their features into + the PMTiles archive, writes layer footprints (internal COGs and PMTiles-vector + uploads) to a companion GeoJSON rendered client-side, and atomically publishes + the archive + companion GeoJSON + a sidecar manifest into the shared public + asset area. Fails loudly and leaves the previous files untouched on any error. + Guarded by a Redis lock so runs can't overlap. + + Runs nightly via celery beat; on a fresh environment app startup also enqueues + one build if no manifest exists yet (see on_startup in app.py). To regenerate + manually (the Redis lock keeps it from colliding with the nightly): + + # enqueue onto the heavy worker (returns immediately) + docker exec -it geoapi_workers python -c \ + "from geoapi.tasks.published_ds_maps import \ + generate_published_ds_maps_pmtiles as t; t.delay()" + + # or run synchronously (blocks until the archive is built) + docker exec -it geoapi_workers python -c \ + "from geoapi.tasks.published_ds_maps import \ + generate_published_ds_maps_pmtiles as t; t.apply()" + + For a non-destructive dry run that only measures counts/size, see + geoapi/misc/dry_run_published_ds_maps.py. + """ + start_time = time.time() + redis_client = redis.Redis(host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=0) + lock = redis_client.lock( + GENERATION_LOCK_KEY, timeout=GENERATION_LOCK_TIMEOUT_SECONDS + ) + if not lock.acquire(blocking=False): + logger.info( + "Public pmtiles generation is already running; skipping this trigger." + ) + return + + try: + with create_task_session() as session: + published_maps = PublishedMapsService.get_published_maps(session) + if not published_maps: + logger.info( + "No published maps for deployment '%s'; leaving any existing " + "archive and manifest untouched.", + settings.APP_ENV, + ) + return + features, layer_features, stats = PublishedMapsExportService.build_features( + session, published_maps + ) + + if not features: + logger.info( + "Published maps yielded no features; leaving any existing archive " + "and manifest untouched." + ) + return + + os.makedirs(public_asset_dir(), exist_ok=True) + # Work under ASSETS_BASE_DIR/tmp so the finished files are on the SAME + # filesystem as the public dir and os.replace() is atomic. + work_dir = tempfile.mkdtemp( + prefix="published_ds_maps_", dir=str(get_temp_dir()) + ) + now = _utc_now() + filename = _archive_filename(now) + layers_filename = _layers_filename(now) + try: + layer_files = _write_layer_files(features, work_dir) + staged_archive = os.path.join(work_dir, filename) + written_count = TippecanoeService.geojson_layers_to_pmtiles( + layer_files, staged_archive + ) + _verify_feature_count(len(features), written_count) + + # atomic within the same filesystem: a partial file is never readable + os.replace(staged_archive, os.path.join(public_asset_dir(), filename)) + _write_public_file( + {"type": "FeatureCollection", "features": layer_features}, + layers_filename, + work_dir, + ) + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + manifest = { + "url": _public_asset_url(filename), + "layers_url": _public_asset_url(layers_filename), + "generated_at": now.isoformat(), + "feature_count": stats["total"], + "cog_count": stats["cog_count"], + "vector_count": stats["vector_count"], + "project_count": stats["project_count"], + "bounds": _bounds(features + layer_features), + "zoom_min": PUBLISHED_DS_MAPS_MIN_ZOOM, + "zoom_max": PUBLISHED_DS_MAPS_MAX_ZOOM, + "schema_version": SCHEMA_VERSION, + } + _write_manifest(manifest) + + # prune only after the manifest points at the new files + _prune_old_public_files({filename, layers_filename}) + + logger.info( + "Public archive published: %s (%s features) + %s (%s COGs, %s vectors) " + "across %s projects in %.1fs.", + filename, + stats["total"], + layers_filename, + stats["cog_count"], + stats["vector_count"], + stats["project_count"], + time.time() - start_time, + ) + except Exception: + logger.exception( + "Failed to generate public pmtiles archive; previous archive and " + "manifest left untouched." + ) + raise + finally: + try: + lock.release() + except Exception: # noqa: E722 + # lock may have already expired; nothing actionable + pass + + +def _write_manifest(manifest: dict) -> None: + """Atomically replace the sidecar manifest.json.""" + tmp_path = manifest_path() + ".tmp" + with open(tmp_path, "w") as f: + json.dump(manifest, f) + os.replace(tmp_path, manifest_path()) + + +def _write_public_file(data: dict, filename: str, work_dir: str) -> None: + """Atomically place a JSON file into the public asset dir. Staged in work_dir + (same filesystem as the public dir), so os.replace() is atomic.""" + staged = os.path.join(work_dir, filename) + with open(staged, "w") as f: + json.dump(data, f) + os.replace(staged, os.path.join(public_asset_dir(), filename)) + + +def enqueue_generation_if_missing() -> bool: + """Enqueue a build if no manifest exists yet. + + Called on app startup so a fresh environment produces an archive within one + task run instead of waiting for the nightly beat. + + :return: True if a generation task was enqueued + """ + if read_manifest() is not None: + return False + try: + generate_published_ds_maps_pmtiles.delay() + logger.info("No public pmtiles manifest present; enqueued initial generation.") + return True + except Exception: + logger.exception("Failed to enqueue initial public pmtiles generation.") + return False diff --git a/geoapi/tests/api_tests/test_published_ds_maps_routes.py b/geoapi/tests/api_tests/test_published_ds_maps_routes.py new file mode 100644 index 00000000..ac136dbc --- /dev/null +++ b/geoapi/tests/api_tests/test_published_ds_maps_routes.py @@ -0,0 +1,28 @@ +from unittest.mock import patch + +ROUTE = "/designsafe/published-maps/pmtiles" + + +def test_get_manifest_returns_200_when_present(test_client): + manifest = { + "url": "https://example.org/assets/public/published_ds_maps_x.pmtiles", + "generated_at": "2026-07-24T03:00:00+00:00", + "feature_count": 5, + "project_count": 2, + "bounds": [-98.0, 30.0, -96.0, 31.0], + "zoom_min": 6, + "zoom_max": 16, + "schema_version": 1, + } + # unauthenticated (guest) access, no token + with patch("geoapi.routes.published_ds_maps.read_manifest", return_value=manifest): + resp = test_client.get(ROUTE) + assert resp.status_code == 200 + assert resp.json() == manifest + + +def test_get_manifest_returns_404_when_missing(test_client): + with patch("geoapi.routes.published_ds_maps.read_manifest", return_value=None): + resp = test_client.get(ROUTE) + assert resp.status_code == 404 + assert "detail" in resp.json() diff --git a/geoapi/tests/api_tests/test_vectors_service.py b/geoapi/tests/api_tests/test_vectors_service.py index 86b404a1..14c339c4 100644 --- a/geoapi/tests/api_tests/test_vectors_service.py +++ b/geoapi/tests/api_tests/test_vectors_service.py @@ -46,25 +46,6 @@ def test_convert_to_geojson_missing_shapefile_additional_files(shapefile_fixture VectorService.convert_to_geojson(shapefile_fixture, additional_files=[]) -@pytest.mark.worker -def test_convert_to_geojson_large_extent_shapefile( - shapefile_large_extent_fixture, shapefile_large_extent_additional_files_fixture -): - geojson_path, bbox = VectorService.convert_to_geojson( - shapefile_large_extent_fixture, - additional_files=shapefile_large_extent_additional_files_fixture, - ) - try: - assert os.path.isfile(geojson_path) - # spans essentially the whole longitude range (antimeridian-crossing) - assert bbox["minx"] <= -179 and bbox["maxx"] >= 179 - gdf = gpd.read_file(geojson_path) - assert gdf.crs.to_epsg() == 4326 - assert len(gdf) == 10 - finally: - shutil.rmtree(os.path.dirname(geojson_path), ignore_errors=True) - - @pytest.mark.worker def test_point_and_polygon_geojson_tiles_retain_geometry( point_and_polygon_geojson_fixture, diff --git a/geoapi/tests/custom/designsafe/test_published_export.py b/geoapi/tests/custom/designsafe/test_published_export.py new file mode 100644 index 00000000..1c69b877 --- /dev/null +++ b/geoapi/tests/custom/designsafe/test_published_export.py @@ -0,0 +1,192 @@ +import uuid + +from geoalchemy2.shape import from_shape +from shapely.geometry import Point, box + +from geoapi.models import Feature, FeatureAsset, Project, TileServer +from geoapi.custom.designsafe.published_export import PublishedMapsExportService +from geoapi.custom.designsafe.published_maps import PublishedMap + + +def _project(db_session): + project = Project(name="pub", description="pub", tenant_id="test", public=True) + db_session.add(project) + db_session.commit() + return project + + +def _point_feature(db_session, project): + feature = Feature(project_id=project.id, properties={}) + feature.the_geom = from_shape(Point(-97.0, 30.0), srid=4326) + db_session.add(feature) + db_session.commit() + return feature + + +def _image_feature(db_session, project): + feature = Feature(project_id=project.id, properties={}) + feature.the_geom = from_shape(Point(-96.0, 31.0), srid=4326) + asset_uuid = uuid.uuid4() + asset = FeatureAsset( + uuid=asset_uuid, + asset_type="image", + path=f"{project.id}/{asset_uuid}.jpeg", + feature=feature, + ) + feature.assets.append(asset) + db_session.add(feature) + db_session.commit() + return feature + + +def _vector_feature(db_session, project): + feature = Feature(project_id=project.id, properties={}) + feature.the_geom = from_shape(box(-99.0, 29.0, -98.0, 30.0), srid=4326) + asset_uuid = uuid.uuid4() + asset = FeatureAsset( + uuid=asset_uuid, + asset_type="vector", + path=f"{project.id}/{asset_uuid}.pmtiles", + feature=feature, + ) + feature.assets.append(asset) + db_session.add(feature) + db_session.commit() + return feature + + +def _cog_tile_server(db_session, project): + ts = TileServer( + project_id=project.id, + name="my-cog", + type="xyz", + kind="cog", + internal=True, + uuid=uuid.uuid4(), + url=f"/assets/{project.id}/{uuid.uuid4()}.cog.tif", + attribution="TACC", + tileOptions={ + "minZoom": 0, + "maxZoom": 22, + "maxNativeZoom": 22, + "bounds": [[30.0, -97.0], [31.0, -96.0]], + "renderOptions": {"colormap_name": "terrain"}, + }, + uiOptions={"opacity": 1, "renderOptions": {"colormap_name": "terrain"}}, + ) + db_session.add(ts) + db_session.commit() + return ts + + +def _published(project, doi="10.17603/ds2-test"): + return PublishedMap( + project=project, + map_uuid=str(project.uuid), + map_name="My Map", + map_path="/", + designsafe_project_id="PRJ-1", + designsafe_project_title="Published One", + designsafe_doi=doi, + ) + + +def test_build_features_tiles_markers_and_lists_layer_footprints(db_session): + project = _project(db_session) + point = _point_feature(db_session, project) + image = _image_feature(db_session, project) + vector = _vector_feature(db_session, project) + _cog_tile_server(db_session, project) + + features, layer_features, stats = PublishedMapsExportService.build_features( + db_session, [_published(project)] + ) + + # markers are tiled; COG + vector become layer footprints (companion geojson) + assert stats == { + "feature_count": 2, + "cog_count": 1, + "vector_count": 1, + "skipped_count": 0, + "project_count": 1, + "total": 2, + } + assert len(features) == 2 + assert len(layer_features) == 2 + + by_type = {} + for f in features: + by_type.setdefault(f["properties"]["feature_type"], []).append(f) + layers_by_type = {} + for f in layer_features: + layers_by_type.setdefault(f["properties"]["feature_type"], []).append(f) + + # shared project context on everything (tiled + layer footprints) + for f in features + layer_features: + props = f["properties"] + assert props["hazmapper_project_uuid"] == str(project.uuid) + assert props["hazmapper_project_name"] == "pub" + assert props["ds_project_name"] == "Published One" + assert "designsafe.storage.published/PRJ-1" in props["ds_project_url"] + assert props["hazmapper_url"].endswith(f"/project-public/{project.uuid}") + assert "selectedFeature" not in props["hazmapper_url"] + for gone in ( + "project_uuid", + "ds_project_id", + "ds_doi", + "has_assets", + "created_date", + ): + assert gone not in props + + assert by_type["point"][0]["properties"]["feature_id"] == point.id + assert "thumbnail_url" not in by_type["point"][0]["properties"] + assert by_type["image"][0]["properties"]["feature_id"] == image.id + + # COG footprint: filled polygon + tile url + descriptor object + bounds + cog = layers_by_type["cog"][0] + assert cog["geometry"]["type"] == "Polygon" + assert cog["properties"]["feature_id"] is None + assert len(cog["properties"]["bounds"]) == 4 + cog_url = cog["properties"]["cog_url"] + assert "/tiles/cog/tiles/WebMercatorQuad/{z}/{x}/{y}.png?url=" in cog_url + assert "colormap_name=terrain" in cog_url + descriptor = cog["properties"]["tile_layer"] + assert descriptor["internal"] is True + assert descriptor["kind"] == "cog" + assert descriptor["tileOptions"]["maxZoom"] == 22 + + # vector footprint: bbox polygon + pointer to its own PMTiles + vec = layers_by_type["vector"][0] + assert vec["geometry"]["type"] == "Polygon" + assert vec["properties"]["feature_id"] == vector.id + assert vec["properties"]["pmtiles_url"].endswith(".pmtiles") + assert "/assets/" in vec["properties"]["pmtiles_url"] + assert len(vec["properties"]["bounds"]) == 4 + + +def test_external_tile_layers_are_not_exported(db_session): + project = _project(db_session) + # an external (user-attached) layer -- must be excluded + external = TileServer( + project_id=project.id, + name="external-basemap", + type="tms", + kind=None, + internal=False, + url="https://example.com/{z}/{x}/{y}.png", + attribution="someone else", + tileOptions={"bounds": [[30.0, -97.0], [31.0, -96.0]]}, + uiOptions={}, + ) + db_session.add(external) + db_session.commit() + + features, layer_features, stats = PublishedMapsExportService.build_features( + db_session, [_published(project)] + ) + + assert stats["cog_count"] == 0 + assert stats["vector_count"] == 0 + assert features == [] + assert layer_features == [] diff --git a/geoapi/tests/custom/designsafe/test_published_maps.py b/geoapi/tests/custom/designsafe/test_published_maps.py new file mode 100644 index 00000000..cc7a4f8a --- /dev/null +++ b/geoapi/tests/custom/designsafe/test_published_maps.py @@ -0,0 +1,126 @@ +import logging +import uuid +from unittest.mock import patch + +from geoapi.models import Project +from geoapi.custom.designsafe.published_maps import PublishedMapsService +from geoapi.settings import settings + + +def _make_project(db_session, name, public): + """Create + commit a project and return it (uuid is auto-generated).""" + project = Project(name=name, description=name, tenant_id="test", public=public) + db_session.add(project) + db_session.commit() + return project + + +def _fake_designsafe_api(pages, details): + """Build a _get_json replacement that routes by URL. + + :param pages: dict of offset -> list result page + :param details: dict of PRJ id -> hazmapperMaps list + """ + + def _get_json(url): + if "/publications/v2?" in url or url.endswith("/publications/v2"): + # extract offset from the query string + offset = int(url.split("offset=")[1].split("&")[0]) + return {"result": pages.get(offset, [])} + designsafe_project_id = url.rsplit("/", 1)[1] + return { + "baseProject": {"hazmapperMaps": details.get(designsafe_project_id, [])} + } + + return _get_json + + +def test_get_published_maps_selects_only_matching_maps_and_logs_exclusions( + db_session, caplog +): + """A map is selected only if it passes all three conditions -- published, + deployment == APP_ENV, and the local project is public -- and the two + inconsistency paths (non-public, not-in-DB) are logged.""" + this_deployment = settings.APP_ENV + + public_project = _make_project(db_session, "public-map", public=True) + private_project = _make_project(db_session, "private-map", public=False) + missing_uuid = str(uuid.uuid4()) + + pages = { + 0: [ + {"projectId": "PRJ-1", "title": "Published One"}, + {"projectId": "PRJ-2", "title": "Published Two"}, + {"projectId": "PRJ-3", "title": "Published Three"}, + {"projectId": "PRJ-4", "title": "Published Four"}, + ] + } + details = { + # selected: published + this deployment + public + "PRJ-1": [ + { + "uuid": str(public_project.uuid), + "name": "Public Map", + "path": "/", + "deployment": this_deployment, + } + ], + # excluded: resolves to a non-public project -> error logged + "PRJ-2": [{"uuid": str(private_project.uuid), "deployment": this_deployment}], + # excluded: registered by a different deployment tier + "PRJ-3": [{"uuid": str(public_project.uuid), "deployment": "some-other-tier"}], + # excluded: this deployment, but no such project in the DB -> warning + "PRJ-4": [{"uuid": missing_uuid, "deployment": this_deployment}], + } + + with patch.object( + PublishedMapsService, + "_get_json", + side_effect=_fake_designsafe_api(pages, details), + ), patch.object( + PublishedMapsService, "inter_request_sleep_seconds", 0 + ), caplog.at_level( + logging.WARNING + ): + selected = PublishedMapsService.get_published_maps(db_session) + + # only the public, this-deployment map survives all three conditions + assert len(selected) == 1 + result = selected[0] + assert result.project.id == public_project.id + assert result.map_uuid == str(public_project.uuid) + assert result.map_name == "Public Map" + assert result.designsafe_project_id == "PRJ-1" + assert result.designsafe_project_title == "Published One" + + # the non-public inconsistency is logged at error level (not raised) + assert any( + record.levelno == logging.ERROR and "non-public" in record.getMessage() + for record in caplog.records + ) + # the missing map is logged at warning level + assert any( + record.levelno == logging.WARNING and missing_uuid in record.getMessage() + for record in caplog.records + ) + + +def test_selector_returns_empty_when_no_deployment_matches(db_session): + """A tier with no published maps of its own yields an empty selection.""" + public_project = _make_project(db_session, "public-map", public=True) + + pages = {0: [{"projectId": "PRJ-1", "title": "Published One"}]} + details = { + "PRJ-1": [ + {"uuid": str(public_project.uuid), "deployment": "a-different-deployment"} + ] + } + + with patch.object( + PublishedMapsService, + "_get_json", + side_effect=_fake_designsafe_api(pages, details), + ), patch.object(PublishedMapsService, "inter_request_sleep_seconds", 0): + selected = PublishedMapsService.get_published_maps(db_session) + + assert selected == [] diff --git a/geoapi/tests/services/test_tippecanoe_service.py b/geoapi/tests/services/test_tippecanoe_service.py index ff607cc9..4f52bf69 100644 --- a/geoapi/tests/services/test_tippecanoe_service.py +++ b/geoapi/tests/services/test_tippecanoe_service.py @@ -82,6 +82,82 @@ def _point_counts_by_zoom(pmtiles_path): return counts +def test_geojson_layers_to_pmtiles_builds_command(): + with patch("geoapi.services.tippecanoe.subprocess.run") as mock_run: + mock_run.return_value = _completed(stderr="42 features, 1234 bytes of geometry") + count = TippecanoeService.geojson_layers_to_pmtiles( + [("points", "/tmp/points.geojson"), ("cog", "/tmp/cog.geojson")], + "/tmp/out.pmtiles", + ) + + # the feature count is parsed from tippecanoe's summary line + assert count == 42 + cmd = mock_run.call_args[0][0] + assert cmd[0] == "tippecanoe" + assert "-o" in cmd and cmd[cmd.index("-o") + 1] == "/tmp/out.pmtiles" + assert cmd[cmd.index("-Z") + 1] == "2" + assert cmd[cmd.index("-z") + 1] == "14" + assert cmd[cmd.index("-B") + 1] == "8" + assert "--drop-densest-as-needed" in cmd + assert "--no-tile-size-limit" not in cmd + assert "--no-feature-limit" in cmd + assert "--no-tiny-polygon-reduction" in cmd + assert "--drop-rate=1" not in cmd + assert "points:/tmp/points.geojson" in cmd + assert "cog:/tmp/cog.geojson" in cmd + + +def test_geojson_layers_to_pmtiles_requires_layers(): + with pytest.raises(ValueError, match="at least one layer"): + TippecanoeService.geojson_layers_to_pmtiles([], "/tmp/out.pmtiles") + + +def test_parse_written_feature_count_ignores_progress_line(): + stderr = "Read 0.00 million features\n7 features, 10 bytes of geometry\n" + assert TippecanoeService._parse_written_feature_count(stderr) == 7 + assert TippecanoeService._parse_written_feature_count("no summary here") is None + + +@pytest.mark.worker +def test_geojson_layers_to_pmtiles_multilayer(): + # tile a point layer and a polygon (COG-footprint-like) layer into one + # archive; assert the reported count matches input and both layers survive. + out_dir = tempfile.mkdtemp(prefix="geoapi_published_ds_maps_test_") + try: + points_path = os.path.join(out_dir, "points.geojson") + shapes_path = os.path.join(out_dir, "shapes.geojson") + with open(points_path, "w") as f: + f.write( + '{"type":"Feature","geometry":{"type":"Point",' + '"coordinates":[-97.7,30.3]},"properties":{"feature_type":"point"}}\n' + ) + with open(shapes_path, "w") as f: + f.write( + '{"type":"Feature","geometry":{"type":"Polygon","coordinates":' + "[[[-97.8,30.2],[-97.6,30.2],[-97.6,30.4],[-97.8,30.4],[-97.8,30.2]]]}," + '"properties":{"feature_type":"cog"}}\n' + ) + + pmtiles_path = os.path.join(out_dir, "out.pmtiles") + count = TippecanoeService.geojson_layers_to_pmtiles( + [("points", points_path), ("cogs", shapes_path)], pmtiles_path + ) + + assert os.path.isfile(pmtiles_path) + assert count == 2 # both input features present, none dropped + + decoded = subprocess.run( + ["tippecanoe-decode", pmtiles_path], + capture_output=True, + text=True, + check=True, + ).stdout + assert '"points"' in decoded + assert '"cogs"' in decoded + finally: + shutil.rmtree(out_dir, ignore_errors=True) + + @pytest.mark.worker def test_geojson_to_pmtiles_keeps_features_at_all_zooms( points_1000_geojson_path_fixture, diff --git a/geoapi/tests/tasks_tests/test_published_ds_maps.py b/geoapi/tests/tasks_tests/test_published_ds_maps.py new file mode 100644 index 00000000..4d32625e --- /dev/null +++ b/geoapi/tests/tasks_tests/test_published_ds_maps.py @@ -0,0 +1,287 @@ +import json +import os +import time +from unittest.mock import MagicMock, patch + +import pytest + +from geoapi.settings import settings +from geoapi.tasks import published_ds_maps +from geoapi.tasks.published_ds_maps import ( + ARCHIVE_PREFIX, + ARCHIVE_SUFFIX, + LAYERS_SUFFIX, + generate_published_ds_maps_pmtiles, +) + +# tiled features (points/shapes); layer footprints go to the companion geojson +FEATURES = [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [-97.7, 30.3]}, + "properties": {"feature_type": "point", "feature_id": 1}, + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-97.8, 30.2], + [-97.6, 30.2], + [-97.6, 30.4], + [-97.8, 30.4], + [-97.8, 30.2], + ] + ], + }, + "properties": {"feature_type": "shape", "feature_id": 2}, + }, +] +# layer footprints -> companion geojson (COG extends the western bound to -98.0) +LAYER_FEATURES = [ + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-98.0, 30.0], + [-97.9, 30.0], + [-97.9, 30.1], + [-98.0, 30.1], + [-98.0, 30.0], + ] + ], + }, + "properties": {"feature_type": "cog", "feature_id": None}, + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-97.9, 30.0], + [-97.8, 30.0], + [-97.8, 30.1], + [-97.9, 30.1], + [-97.9, 30.0], + ] + ], + }, + "properties": {"feature_type": "vector", "feature_id": 3}, + }, +] +STATS = { + "feature_count": 2, + "cog_count": 1, + "vector_count": 1, + "project_count": 1, + "total": 2, +} + + +def _write_dummy_archive(layers, output_path, *args, **kwargs): + with open(output_path, "wb") as f: + f.write(b"PMTILES-DUMMY") + return len(FEATURES) # tippecanoe "wrote" the 2 tiled features + + +class _ContextManagerMock: + def __enter__(self): + return MagicMock() + + def __exit__(self, *exc): + return False + + +def _patchers( + isolated_assets_dir, + published_maps=("sentinel",), + build_return=(FEATURES, LAYER_FEATURES, STATS), + tile_side_effect=_write_dummy_archive, + lock_acquired=True, +): + """Common patch set. Returns a list of started patchers (caller stops them).""" + lock = MagicMock() + lock.acquire.return_value = lock_acquired + redis_client = MagicMock() + redis_client.lock.return_value = lock + + patchers = [ + patch.object(settings, "ASSETS_BASE_DIR", isolated_assets_dir), + patch.object(published_ds_maps.redis, "Redis", return_value=redis_client), + patch.object( + published_ds_maps, "create_task_session", return_value=_ContextManagerMock() + ), + patch.object( + published_ds_maps.PublishedMapsService, + "get_published_maps", + return_value=list(published_maps), + ), + patch.object( + published_ds_maps.PublishedMapsExportService, + "build_features", + return_value=build_return, + ), + patch.object( + published_ds_maps.TippecanoeService, + "geojson_layers_to_pmtiles", + side_effect=tile_side_effect, + ), + ] + return patchers, redis_client + + +def _run(patchers): + for p in patchers: + p.start() + try: + return generate_published_ds_maps_pmtiles() + finally: + for p in reversed(patchers): + p.stop() + + +def test_generate_writes_archive_and_manifest(tmp_path): + assets_dir = str(tmp_path) + patchers, _ = _patchers(assets_dir) + _run(patchers) + + public_dir = os.path.join(assets_dir, "public") + archives = [ + n + for n in os.listdir(public_dir) + if n.startswith(ARCHIVE_PREFIX) and n.endswith(ARCHIVE_SUFFIX) + ] + layers = [n for n in os.listdir(public_dir) if n.endswith(LAYERS_SUFFIX)] + assert len(archives) == 1 + assert len(layers) == 1 + filename, layers_filename = archives[0], layers[0] + + # the companion geojson is a FeatureCollection of the layer footprints + with open(os.path.join(public_dir, layers_filename)) as f: + layers_fc = json.load(f) + assert layers_fc["type"] == "FeatureCollection" + assert len(layers_fc["features"]) == 2 + + with open(os.path.join(public_dir, "manifest.json")) as f: + manifest = json.load(f) + assert manifest["feature_count"] == 2 + assert manifest["cog_count"] == 1 + assert manifest["vector_count"] == 1 + assert manifest["project_count"] == 1 + assert manifest["zoom_min"] == 2 + assert manifest["zoom_max"] == 14 + assert manifest["schema_version"] == 1 + assert manifest["url"].endswith(f"/assets/public/{filename}") + assert manifest["layers_url"].endswith(f"/assets/public/{layers_filename}") + assert len(manifest["bounds"]) == 4 + # bounds span the tiled features AND the COG footprint (COG extends west to -98) + assert manifest["bounds"][0] == pytest.approx(-98.0) + assert manifest["bounds"][2] == pytest.approx(-97.6) + + +def test_generate_skips_when_no_published_maps(tmp_path): + assets_dir = str(tmp_path) + patchers, _ = _patchers(assets_dir, published_maps=()) + _run(patchers) + + # nothing written, no manifest + assert not os.path.exists(os.path.join(assets_dir, "public", "manifest.json")) + + +def test_generate_skips_when_lock_held(tmp_path): + assets_dir = str(tmp_path) + patchers, _ = _patchers(assets_dir, lock_acquired=False) + # build_features must not be called if the lock isn't acquired + with patch.object( + published_ds_maps.PublishedMapsExportService, "build_features" + ) as build: + for p in patchers: + p.start() + try: + generate_published_ds_maps_pmtiles() + finally: + for p in reversed(patchers): + p.stop() + build.assert_not_called() + + +def test_generate_aborts_on_count_mismatch_without_touching_manifest(tmp_path): + assets_dir = str(tmp_path) + + def _wrong_count(layers, output_path, *a, **k): + with open(output_path, "wb") as f: + f.write(b"PMTILES-DUMMY") + return 1 # fewer than the 2 we handed it -> silent drop + + patchers, _ = _patchers(assets_dir, tile_side_effect=_wrong_count) + with pytest.raises(RuntimeError, match="feature count mismatch"): + _run(patchers) + + # no archive placed, no manifest written + public_dir = os.path.join(assets_dir, "public") + assert not os.path.exists(os.path.join(public_dir, "manifest.json")) + placed = [ + n + for n in os.listdir(public_dir) + if n.startswith(ARCHIVE_PREFIX) + and (n.endswith(ARCHIVE_SUFFIX) or n.endswith(LAYERS_SUFFIX)) + ] + assert placed == [] + + +def test_enqueue_generation_if_missing_enqueues_only_when_absent(): + with patch.object( + published_ds_maps, "read_manifest", return_value={"url": "x"} + ), patch.object( + published_ds_maps.generate_published_ds_maps_pmtiles, "delay" + ) as delay: + assert published_ds_maps.enqueue_generation_if_missing() is False + delay.assert_not_called() + + with patch.object( + published_ds_maps, "read_manifest", return_value=None + ), patch.object( + published_ds_maps.generate_published_ds_maps_pmtiles, "delay" + ) as delay: + assert published_ds_maps.enqueue_generation_if_missing() is True + delay.assert_called_once() + + +def test_generate_prunes_old_archives_but_keeps_recent(tmp_path): + assets_dir = str(tmp_path) + public_dir = os.path.join(assets_dir, "public") + os.makedirs(public_dir, exist_ok=True) + + old = os.path.join(public_dir, f"{ARCHIVE_PREFIX}20200101T000000Z{ARCHIVE_SUFFIX}") + old_layers = os.path.join( + public_dir, f"{ARCHIVE_PREFIX}20200101T000000Z{LAYERS_SUFFIX}" + ) + recent = os.path.join( + public_dir, f"{ARCHIVE_PREFIX}20990101T000000Z{ARCHIVE_SUFFIX}" + ) + for path in (old, old_layers, recent): + with open(path, "wb") as f: + f.write(b"old") + # make the old files older than the prune window; leave `recent` fresh + stale = time.time() - (30 * 3600) + os.utime(old, (stale, stale)) + os.utime(old_layers, (stale, stale)) + + patchers, _ = _patchers(assets_dir) + _run(patchers) + + remaining = set(os.listdir(public_dir)) + assert os.path.basename(old) not in remaining # stale .pmtiles pruned + assert os.path.basename(old_layers) not in remaining # stale companion pruned + assert os.path.basename(recent) in remaining # kept (fresh) + # the newly generated archive is present too + assert any( + n.startswith(ARCHIVE_PREFIX) + and n.endswith(ARCHIVE_SUFFIX) + and n not in {os.path.basename(old), os.path.basename(recent)} + for n in remaining + ) diff --git a/geoapi/utils/client_backend.py b/geoapi/utils/client_backend.py index 7c415a29..64d5a8d1 100644 --- a/geoapi/utils/client_backend.py +++ b/geoapi/utils/client_backend.py @@ -80,3 +80,28 @@ def get_deployed_geoapi_url(): else: logger.exception(f"Unknown/unsupported APP_ENV:{settings.APP_ENV}") raise ApiException(f"Unknown APP_ENV:{settings.APP_ENV}") + + +def get_deployed_hazmapper_url(): + """ + Get the Hazmapper frontend base URL for the current environment. + + Used to build public-map deep links, e.g. + ``{base}/project-public/{uuid}?selectedFeature={id}``. + """ + hazmapper_urls = { + "local": "http://localhost:4200", + "production": "https://hazmapper.tacc.utexas.edu/hazmapper", + "staging": "https://hazmapper.tacc.utexas.edu/staging", + "dev": "https://hazmapper.tacc.utexas.edu/dev", + "testing": "http://localhost:4200", + # Next 3 entries are for proxmox-migration. Remove when completed. https://tacc-main.atlassian.net/browse/WG-615 + "production-tmp": "https://hazmapper.tacc.utexas.edu/hazmapper-tmp", + "staging-tmp": "https://hazmapper.tacc.utexas.edu/staging-tmp", + "dev-tmp": "https://hazmapper.tacc.utexas.edu/dev-tmp", + } + if settings.APP_ENV in hazmapper_urls: + return hazmapper_urls[settings.APP_ENV] + else: + logger.exception(f"Unknown/unsupported APP_ENV:{settings.APP_ENV}") + raise ApiException(f"Unknown APP_ENV:{settings.APP_ENV}")