Merge pull request #223 from UW-Macrostrat/legacy-tileserver-update

Legacy tileserver update

Author: davenquinn

.idea/data_source_mapping.xml

@@ -113,7 +113,7 @@ localhost, macrostrat.local {
     }
 
     handle_path /tiles/* {
-        reverse_proxy tileserver:8000 {
+        reverse_proxy tileserver_cache:8000 {
             header_down -Access-Control-Allow-Origin
             header_down -Access-Control-Allow-Credentials
             header_down -Access-Control-Allow-Methods
@@ -148,3 +148,21 @@ storage-ui.macrostrat.local {
     tls internal
     reverse_proxy storage:9001
 }
+
+tiles.macrostrat.local {
+    tls internal
+    reverse_proxy tileserver_cache:8000 {
+        header_down -Access-Control-Allow-Origin
+        header_down -Access-Control-Allow-Credentials
+        header_down -Access-Control-Allow-Methods
+    }
+
+    # Local CORS headers
+
+    header {
+      Access-Control-Allow-Origin *
+      Access-Control-Allow-Methods "GET, POST, PUT, PATCH, DELETE, OPTIONS"
+    }
+
+
+}

local-root/Caddyfile

@@ -0,0 +1,70 @@
+vcl 4.1;
+
+backend backend-core {
+    .host = "tileserver_core";
+    .port = "8000";
+}
+
+
+backend backend-legacy {
+    .host = "tileserver_legacy";
+    .port = "8000";
+    # Don't require the connection to be up at the start
+}
+
+
+# Bypass Varnish if tile has a cache=bypass query parameter
+sub vcl_recv {
+    # Set the backend hints to route to the correct upstream
+    # For now the index.html route is served from the legacy tileserver
+    if (req.url ~ "^/$" || req.url ~ "^/preview$") {
+        set req.backend_hint = backend-legacy;
+    } else if (req.url ~ ".*\.(png|mvt)$") {
+        # Png and mvt tiles are served from the legacy tileserver
+        set req.backend_hint = backend-legacy;
+    } else {
+        # Everything else is served from the core tileserver
+        set req.backend_hint = backend-core;
+        # TODO: could add a v3 prefix here if needed
+    }
+
+    # Bypass the cache for zoom levels greater than 14
+    if (req.url ~ ".*\/([0-9][56789]|[0-9]{3,})\/[0-9]+\/[0-9]+(\.png|\.mvt)?") {
+        set req.http.X-Cache = "bypass";
+        return (pass);
+    }
+
+
+    if (req.url ~ ".*cache=bypass.*") {
+        set req.http.X-Cache = "bypass";
+        return (pass);
+    }
+
+    # If has no-cache header, bypass Varnish
+    # This might be redundant, I'm not quite sure what the default behavior is
+    if (req.http.Cache-Control ~ "no-cache") {
+        set req.http.X-Cache = "bypass";
+        return (pass);
+    }
+}
+
+sub vcl_deliver {
+    if (obj.hits > 0) {
+        # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed
+        set resp.http.X-Cache = "hit";
+    } else {
+        set resp.http.X-Cache = "miss";
+    }
+}
+
+sub vcl_backend_response {
+    # Set a long TTL for tiles
+    if (bereq.url ~ ".*\.(png|mvt)$") {
+        set beresp.ttl = 1d;
+        # Allow stale content while revalidating
+        set beresp.grace = 5m;
+    } else {
+        # Shorter TTL for other content
+        set beresp.ttl = 5m;
+    }
+}

local-root/configs/tileserver-cache.vcl

@@ -6,7 +6,7 @@ services:
     volumes:
       - ./Caddyfile:/etc/caddy/Caddyfile
     labels:
-      - dev.orbstack.domains=macrostrat.local,storage.macrostrat.local,storage-ui.macrostrat.local,dev.macrostrat.local
+      - dev.orbstack.domains=macrostrat.local,storage.macrostrat.local,storage-ui.macrostrat.local,dev.macrostrat.local,tiles.macrostrat.local
   database:
     # PostgreSQL 13 is needed in order to allow force-dropping the database
     # (in testing mode)
@@ -78,7 +78,13 @@ services:
       # Perhaps we can put this config in the database?
       - PGRST_DB_AGGREGATES_ENABLED=true
   # Tileserver
-  tileserver:
+  tileserver_cache:
+    image: varnish:7.7
+    environment:
+      - VARNISH_HTTP_PORT=8000
+    volumes:
+      - ./configs/tileserver-cache.vcl:/etc/varnish/default.vcl
+  tileserver_core:
     image: hub.opensciencegrid.org/macrostrat/tileserver:main
     build:
       context: ../services/tileserver
@@ -95,26 +101,25 @@ services:
     command: >
       uv run uvicorn --host 0.0.0.0 --port 8000
         macrostrat.tileserver:app
-        --root-path /tiles
         --log-level debug
         --proxy-headers
         --forwarded-allow-ips=*
-  legacy_tileserver:
+  tileserver_legacy:
     image: hub.opensciencegrid.org/macrostrat/legacy-tileserver:main
     build:
       context: ../services/legacy-tileserver
       additional_contexts:
         py-modules: ../py-modules
     environment:
       - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DB}
+      - MAPNIK_POOL_SIZE=64
     command: >
       uvicorn --host 0.0.0.0 --port 8000
         macrostrat.legacy_tileserver:app
-        --root-path /tiles/legacy
         --log-level debug
         --proxy-headers
         --forwarded-allow-ips=*
-  # Storage
+  # Storage system
   storage:
     image: minio/minio:RELEASE.2024-12-18T13-15-44Z-cpuv1
     environment:
@@ -132,7 +137,7 @@ services:
     build: ${MACROSTRAT_WEB_SRC:-""}
     environment:
       - VITE_MAPBOX_API_TOKEN=${MAPBOX_TOKEN}
-      - VITE_MACROSTRAT_TILESERVER_DOMAIN=https://macrostrat.local/tiles
+      - VITE_MACROSTRAT_TILESERVER_DOMAIN=https://tiles.macrostrat.local
       - VITE_MACROSTRAT_API_DOMAIN=https://macrostrat.local
       # Needed for server-side rendering requests to not fail on self-signed certs (which OrbStack provides)
       - NODE_TLS_REJECT_UNAUTHORIZED=0

local-root/docker-compose.yaml

@@ -4,18 +4,22 @@
 
 FROM python:3.9@sha256:c0dcc146710fed0a6d62cb55b92f00bfbfc3b931fff6218f4958bab58333c37b
 
-ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    POETRY_NO_INTERACTION=1 \
-    POETRY_CACHE_DIR=/tmp/poetry-cache \
-    POETRY_VIRTUALENVS_CREATE=false
-
-WORKDIR /app
-RUN python3 -m venv /venv
-ENV PATH="/venv/bin:$PATH"
-ENV VIRTUAL_ENV=/venv
-
-# MAPNIK
-# Install mapnik for compiling legacy image tiles
+# Install UV for Python package management
+COPY --from=ghcr.io/astral-sh/uv:0.9.9 /uv /uvx /bin/
+
+ENV UV_SYSTEM_PYTHON=0
+ENV UV_LINK_MODE=copy
+ENV UV_COMPILE_BYTECODE=1
+
+WORKDIR /services/main
+
+# Create and enter a virtual environment
+RUN python3 -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+ENV VIRTUAL_ENV="/opt/venv"
+
+# Install MAPNIK
+# Install dependencies for compiling legacy image tiles
 RUN apt-get update -y && \
   apt-get install -y --no-install-recommends \
   build-essential software-properties-common curl \
@@ -32,10 +36,20 @@ RUN cd /tmp/mapnik-v${MAPNIK_VERSION} && python scons/scons.py configure
 RUN cd /tmp/mapnik-v${MAPNIK_VERSION} && make JOBS=4 && make install JOBS=4
 
 ENV BOOST_PYTHON_LIB=boost_python39
-# Python bindings to mapnik
+# Build/install Mapnik Python bindings
 ARG PYTHON_MAPNIK_COMMIT=7da019cf9eb12af8f8aa88b7d75789dfcd1e901b
 RUN mkdir -p /opt/python-mapnik && curl -L https://github.com/mapnik/python-mapnik/archive/${PYTHON_MAPNIK_COMMIT}.tar.gz | tar xz -C /opt/python-mapnik --strip-components=1
-RUN cd /opt/python-mapnik && python3 setup.py install && rm -r /opt/python-mapnik/build
+
+# Carto is needed for CartoCSS -> Mapnik XML conversion
+# Install Node.js and npm
+# Import the Node.js repository GPG key
+RUN apt-get update -y && \
+  apt-get install -y nodejs npm && \
+  npm install -g carto && \
+  rm -rf /tmp/* && \
+  rm -rf /var/lib/apt/lists/*
+
+RUN uv pip install /opt/python-mapnik
 
 # Remove build dependencies
 RUN apt-get remove -y \
@@ -44,43 +58,30 @@ RUN apt-get remove -y \
   libboost-regex-dev libboost-system-dev libboost-thread-dev libicu-dev libtiff5-dev \
   libfreetype-dev libpng-dev libxml2-dev libproj-dev libcairo-dev libharfbuzz-dev python-dev
 
-# CartoCSS stylesheet generation
-# Install nodejs
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \
-  apt-get install -y nodejs && \
-  rm -rf /var/lib/apt/lists/*
-
-# Install carto
-RUN npm install -g carto
-
-# Clear temporary files
-RUN rm -rf /tmp/*
-
-# The rest of this (for the server itself) should be easier.
-
-# Install UV
-COPY --from=ghcr.io/astral-sh/uv:0.9.9 /uv /uvx /bin/
-
-ENV UV_LINK_MODE=copy
-ENV UV_COMPILE_BYTECODE=1
-
 # NEED TO specify a build context for the py-modules directory
 COPY --from=py-modules tileserver-utils /py-modules/tileserver-utils
 
-WORKDIR /services/main
-
-# Install dependencies
+# Install dependencies. The --inexact flag allows us to keep the unmanaged mapnik installation
+# even though it is not specified in the lockfile.
 RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,source=uv.lock,target=uv.lock \
     --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    uv sync --locked --no-install-project --no-editable --active
+    uv sync --inexact --locked --no-install-project --no-editable --active
+
+# We've already built this but we have to reinstall it after uv sync
+RUN uv pip install /opt/python-mapnik && \
+    rm -rf /opt/python-mapnik
 
 # Add source files to project
 ADD . /services/main
 
 # Sync the project
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv sync --locked --no-editable --active
+    uv sync --inexact --locked --no-editable --active
+
+# Check whether Mapnik bindings are properly installed
+RUN python -c "import mapnik; print('mapnik version:', mapnik.mapnik_version())" && \
+    python -c "import uvicorn; print('uvicorn version:', uvicorn.__version__)"
 
 EXPOSE 8000
 

services/legacy-tileserver/Dockerfile

@@ -32,13 +32,19 @@
 @app.on_event("startup")
 async def startup_event():
     """Application startup: register the database connection and create table list."""
-    # Don't rely on poort TimVT handling of database connections
     setup_stderr_logs("macrostrat.legacy_tileserver")
 
     url = environ["DATABASE_URL"]
+
+    # Tile rendering map pool size
+    # This controls how many image tiles can be concurrently rendered.
+    # Database access is somewhat inefficient, so we may need to adjust this.
+    mapnik_pool_size = int(environ.get("MAPNIK_POOL_SIZE", "64"))
+    log.info(f"Setting up Mapnik map pool with size {mapnik_pool_size}")
+
     app.state.pool = await create_pool_b(url)
     db = Database(url)
-    app.state.map_pool = MapnikMapPool(8)
+    app.state.map_pool = MapnikMapPool(mapnik_pool_size)
     await app.state.map_pool.setup(db)
 
 

services/legacy-tileserver/macrostrat/legacy_tileserver/__init__.py

@@ -1,6 +1,6 @@
 [project]
 name = "macrostrat.legacy-tileserver"
-version = "2.3.0-beta.2"
+version = "2.3.0-beta.3"
 description = "Macrostrat legacy tile server"
 authors = [{ name = "Daven Quinn", email = "[email protected]" }]
 requires-python = ">=3.9,<3.10"