Mapnik Datasource pooling and logging

Author: davenquinn

local-root/docker-compose.yaml

@@ -114,7 +114,7 @@ services:
         py-modules: ../py-modules
     environment:
       - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}
-      - MAPNIK_POOL_SIZE=64
+      - MAPNIK_POOL_SIZE=8
     command: >
       uvicorn --host 0.0.0.0 --port 8000
         macrostrat.legacy_tileserver:app

services/legacy-tileserver/macrostrat/legacy_tileserver/__init__.py

@@ -1,16 +1,10 @@
-from enum import Enum
-from functools import lru_cache
 from os import environ
-from pathlib import Path
 
 from buildpg import render
 from buildpg.asyncpg import create_pool_b
+from enum import Enum
 from fastapi import BackgroundTasks, Depends, FastAPI, Request
-from morecantile import Tile
-from starlette.middleware import Middleware
-from starlette.middleware.cors import CORSMiddleware
-from starlette.responses import HTMLResponse
-
+from functools import lru_cache
 from macrostrat.database import Database
 from macrostrat.tileserver_utils import (
     CachedTileArgs,
@@ -20,6 +14,11 @@
     handle_cached_tile_request,
 )
 from macrostrat.utils import get_logger, setup_stderr_logs
+from morecantile import Tile
+from pathlib import Path
+from starlette.middleware import Middleware
+from starlette.middleware.cors import CORSMiddleware
+from starlette.responses import HTMLResponse
 
 from .image_tiles import MapnikMapPool, get_image_tile
 
@@ -39,7 +38,7 @@ async def startup_event():
     # Tile rendering map pool size
     # This controls how many image tiles can be concurrently rendered.
     # Database access is somewhat inefficient, so we may need to adjust this.
-    mapnik_pool_size = int(environ.get("MAPNIK_POOL_SIZE", "64"))
+    mapnik_pool_size = int(environ.get("MAPNIK_POOL_SIZE", "8"))
     log.info(f"Setting up Mapnik map pool with size {mapnik_pool_size}")
 
     app.state.pool = await create_pool_b(url)

services/legacy-tileserver/macrostrat/legacy_tileserver/image_tiles/__init__.py

@@ -41,5 +41,7 @@ async def get_image_tile(request: Request, args: CachedTileArgs) -> bytes:
         # Render map to image
         im = Image(512, 512)
         render(_map, im, 2)
+        
         # Return image as binary
-        return im.tostring("png")
+        res = im.tostring("png")
+        return res

services/legacy-tileserver/macrostrat/legacy_tileserver/image_tiles/pool.py

@@ -69,7 +69,10 @@ async def map_context(self, scale: str) -> Map:
         """Get a map from the pool."""
         q = self.storage[scale]
         _map = await q.get()
+        t = time.time()
         try:
             yield _map
         finally:
             await q.put(_map)
+            dt = time.time() - t
+            log.debug(f"Returned map to pool for scale {scale} in {dt:.3f} seconds")