Merge pull request #413 from headwaymaps/mkirk/update-planet-260420

Switch to pmtiles, update planet data

Author: michaelkirk

bin/_headway_version.sh

@@ -21,7 +21,7 @@
 # A bump in the DATA tag pretty much always implies a bump in the CONTAINER
 # tag, but not necessarily vice-versa.
 
-export HEADWAY_DATA_TAG=0.10.0
+export HEADWAY_DATA_TAG=0.11.0
 
 # # Schema change Log
 #

bin/build

@@ -45,4 +45,4 @@ if [ -d "$TRANSIT_CONFIG_DIR" ]; then
     bin/build-transit "$CONFIG_DIR"
 fi
 
-echo "Next, run: git co . && k8s/bin/regenerate-all"
+echo "Next, run: git co . && ./bin/k8s-regenerate-all"

bin/build-images

@@ -77,21 +77,27 @@ function build_image() {
     echo "missing 'container' arg"
     exit 1
   fi
-  local tag=$3
-  if [ -z "$tag" ]; then
-    echo "missing 'tag' arg"
+  local addresses=$3
+  if [ -z "$addresses" ]; then
+    echo "missing 'addresses' arg"
     exit 1
   fi
 
-  HEADWAY_IMAGE_REPO="${HEADWAY_IMAGE_REPO:-ghcr.io/headwaymaps}"
-  local image_ref="${HEADWAY_IMAGE_REPO}/${container}:${tag}"
-
   if [ "$PUBLISH" = true ]; then
-    echo "Publishing ${image_ref}..."
-    dagger -c "${build_function} | publish \"${image_ref}\""
+    echo "Publishing ${container}: ${addresses}"
+    dagger -c "publish-multi --container=\$(${build_function}) --addresses=${addresses}"
   else
-    echo "Exporting ${image_ref} to local docker..."
-    dagger --interactive -c "${build_function} | export-image \"${image_ref}\""
+    # Dagger's host-image-loader can only fulfill one ExportImage per session,
+    # so export to the first tag and use `docker tag` for the rest. This keeps
+    # all tags pointing at the same image id locally.
+    IFS=',' read -ra _addr_list <<< "$addresses"
+    local primary="${_addr_list[0]}"
+    echo "Exporting ${container} to local docker: ${primary}"
+    dagger --interactive -c "${build_function} | export-image \"${primary}\""
+    for extra in "${_addr_list[@]:1}"; do
+      echo "Tagging ${extra} -> ${primary}"
+      docker tag "$primary" "$extra"
+    done
   fi
 }
 
@@ -108,30 +114,49 @@ function build_image_tags() {
         web_branding_param="--branding=$BRANDING"
     fi
 
-    # Convert comma/space separated tags to array
-    IFS=', ' read -ra tag_array <<< "$tags_input"
+    HEADWAY_IMAGE_REPO="${HEADWAY_IMAGE_REPO:-ghcr.io/headwaymaps}"
 
+    # Convert comma/space separated tags to array, dropping empties
+    IFS=', ' read -ra tag_array <<< "$tags_input"
+    local tags=()
     for tag in "${tag_array[@]}"; do
-        # Skip empty tags
-        if [ -z "$tag" ]; then
-            continue
-        fi
-
-        echo "Processing tag: $tag"
-
-        should_build "otp-init" && build_image "otp-init-container" "opentripplanner-init" "$tag"
-        should_build "otp" && build_image "otp-serve-container" "opentripplanner" "$tag"
-        should_build "pelias-init" && build_image "pelias-init-container" "pelias-init" "$tag"
-        should_build "travelmux-init" && build_image "travelmux-init-container" "travelmux-init" "$tag"
-        should_build "travelmux" && build_image "travelmux-serve-container" "travelmux" "$tag"
-        should_build "tileserver" && build_image "tileserver-serve-container" "tileserver" "$tag"
-        should_build "tileserver-init" && build_image "tileserver-init-container" "tileserver-init" "$tag"
-        should_build "valhalla-init" && build_image "valhalla-init-container" "valhalla-init" "$tag"
-        should_build "valhalla" && build_image "valhalla-serve-container" "valhalla" "$tag"
-        should_build "frontend" && build_image "web-serve-container ${web_branding_param}" "headway" "$tag"
-        should_build "frontend-init" && build_image "web-init-container" "headway-init" "$tag"
-        true
+        [ -n "$tag" ] && tags+=("$tag")
     done
+
+    if [ ${#tags[@]} -eq 0 ]; then
+        echo "no tags provided"
+        exit 1
+    fi
+
+    # Build comma-separated "repo/container:tag" list for one image across all tags.
+    function addresses_for() {
+        local container=$1
+        local out=""
+        for tag in "${tags[@]}"; do
+            out+="${HEADWAY_IMAGE_REPO}/${container}:${tag},"
+        done
+        echo "${out%,}"
+    }
+
+    function build() {
+        local filter=$1
+        local build_function=$2
+        local container=$3
+        should_build "$filter" || return 0
+        build_image "$build_function" "$container" "$(addresses_for "$container")"
+    }
+
+    build "frontend"        "web-serve-container ${web_branding_param}" "headway"
+    build "frontend-init"   "web-init-container"                        "headway-init"
+    build "otp-init"        "otp-init-container"                        "opentripplanner-init"
+    build "otp"             "otp-serve-container"                       "opentripplanner"
+    build "pelias-init"     "pelias-init-container"                     "pelias-init"
+    build "tileserver"      "tileserver-serve-container"                "tileserver"
+    build "tileserver-init" "tileserver-init-container"                 "tileserver-init"
+    build "travelmux-init"  "travelmux-init-container"                  "travelmux-init"
+    build "travelmux"       "travelmux-serve-container"                 "travelmux"
+    build "valhalla-init"   "valhalla-init-container"                   "valhalla-init"
+    build "valhalla"        "valhalla-serve-container"                  "valhalla"
 }
 
 echo "Mode: $([ "$PUBLISH" = true ] && echo "Publish" || echo "Export")"

bin/publish-data

@@ -88,7 +88,7 @@ set -o nounset
 
 upload "${HEADWAY_AREA}.valhalla.tar.zst"      "${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}"
 upload "${HEADWAY_AREA}.elasticsearch.tar.zst" "${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}"
-upload "${HEADWAY_AREA}.mbtiles"               "${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}"
+upload "${HEADWAY_AREA}.pmtiles"               "${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}"
 upload "${HEADWAY_AREA}.placeholder.tar.zst"   "${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}"
 upload "${HEADWAY_AREA}.osm.pbf"               "${HEADWAY_DATA_TAG}/${HEADWAY_AREA_TAG}"
 

bin/reset-services

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Stop, remove volumes, and restart docker compose services
+
+set -e
+
+CONFIG_DIR="$1"
+if [ -z "$CONFIG_DIR" ]; then
+    cat <<EOS
+Usage: $0 <build-config-dir>
+Example: Basic
+    $0 builds/Bogota
+Example: With Transit
+    COMPOSE_FILE=docker-compose-with-transit.yaml $0 builds/Seattle
+EOS
+    exit 1
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+"$SCRIPT_DIR/stop-and-remove-services" "$CONFIG_DIR"
+"$SCRIPT_DIR/start-services" "$CONFIG_DIR"

bin/start-services

@@ -3,10 +3,27 @@
 
 set -e
 
+FOLLOW_LOGS=1
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --no-follow-logs)
+            FOLLOW_LOGS=0
+            shift
+            ;;
+        -*)
+            echo "Unknown option: $1"
+            exit 1
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
 CONFIG_DIR="$1"
 if [ -z "$CONFIG_DIR" ]; then
     cat <<EOS
-Usage: $0 <build-config-dir>
+Usage: $0 [--no-follow-logs] <build-config-dir>
 Example: Basic
     $0 builds/Bogota
 Example: With Transit
@@ -50,3 +67,9 @@ echo "Services started! Use bin/wait-for-services to check readiness."
 echo ""
 echo "To stop services and remove volumes, run:"
 echo "  bin/stop-and-remove-services $CONFIG_DIR"
+echo ""
+
+cd "$APP_ROOT"
+if [ "$FOLLOW_LOGS" -eq 1 ]; then
+    bin/docker-compose "$CONFIG_DIR" logs --follow
+fi

bin/update-planet-version

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -euo pipefail
+
+APP_ROOT=$(git rev-parse --show-toplevel)
+ENV_FILE="$APP_ROOT/builds/planet/.env"
+S3_BASE="s3://osm-planet-us-west-2/planet/pbf"
+
+LATEST_YEAR=$(aws s3 ls --no-sign-request "$S3_BASE/" \
+    | grep -oE '[0-9]{4}/' \
+    | tr -d '/' \
+    | sort -n \
+    | tail -1)
+
+LATEST_FILE=$(aws s3 ls --no-sign-request "$S3_BASE/$LATEST_YEAR/" \
+    | grep '\.osm\.pbf$' \
+    | awk '{print $4}' \
+    | sort \
+    | tail -1)
+
+YYMMDD=$(echo "$LATEST_FILE" | sed 's/planet-\([0-9]*\)\.osm\.pbf/\1/')
+NEW_VERSION="v1.$YYMMDD"
+CURRENT_VERSION=$(grep '^HEADWAY_PLANET_VERSION=' "$ENV_FILE" | cut -d= -f2)
+
+if [ "$NEW_VERSION" = "$CURRENT_VERSION" ]; then
+    echo "Already up to date: $CURRENT_VERSION"
+    exit 0
+fi
+
+echo "Updating HEADWAY_PLANET_VERSION: $CURRENT_VERSION -> $NEW_VERSION"
+
+perl -i -pe "s/^HEADWAY_PLANET_VERSION=.*/HEADWAY_PLANET_VERSION=$NEW_VERSION/" "$ENV_FILE"

builds/planet/.env

@@ -1,6 +1,6 @@
 # See latest releases with: aws s3 ls --no-sign-request s3://osm-planet-us-west-2/planet/pbf/2025/
 # And update the HEADWAY_PLANET_VERSION accordingly. The build scripts will fetch the corresponding osm.pbf
-HEADWAY_PLANET_VERSION=v1.251124
+HEADWAY_PLANET_VERSION=v1.260420
 HEADWAY_AREA=maps-earth-planet-${HEADWAY_PLANET_VERSION}
 HEADWAY_AREA_TAG="$HEADWAY_AREA"
 HEADWAY_COUNTRIES="ALL"

dagger/main.go

@@ -153,11 +153,11 @@ func (h *Headway) Build(ctx context.Context) (*dagger.Directory, error) {
 
 	output = output.WithFile(h.Area+".osm.pbf", h.OSMExport.File)
 
-	mbtiles, err := h.Mbtiles(ctx)
+	pmtiles, err := h.Pmtiles(ctx)
 	if err != nil {
-		return nil, fmt.Errorf("failed to build mbtiles: %w", err)
+		return nil, fmt.Errorf("failed to build pmtiles: %w", err)
 	}
-	output = output.WithFile(h.Area+".mbtiles", mbtiles)
+	output = output.WithFile(h.Area+".pmtiles", pmtiles)
 
 	valhalla := h.ValhallaTiles(ctx)
 	output = output.WithFile(h.Area+".valhalla.tar.zst", valhalla.Compress())
@@ -217,11 +217,11 @@ func martinBinary() *dagger.File {
 	const martinFeatures = "fonts,mbtiles,pmtiles,styles,sprites"
 
 	// To build from source (e.g. for debugging a fork), set this to true
-	const buildFromSource = false
+	const buildFromSource = true
 	if buildFromSource {
 		return rustContainer("git").
 			WithEnvVariable("CACHE_BUSTER", time.Now().String()).
-			WithExec([]string{"git", "clone", "--branch", "mkirk/fix-forwarding-header", "--depth=1", "https://github.com/michaelkirk/martin.git", "/martin"}).
+			WithExec([]string{"git", "clone", "--branch", "mkirk/relative-source-urls", "--depth=1", "https://github.com/michaelkirk/martin.git", "/martin"}).
 			WithWorkdir("/martin").
 			WithExec([]string{"cargo", "build", "--release", "--locked", "--no-default-features", "--features", martinFeatures}).
 			File("target/release/martin")
@@ -244,15 +244,15 @@ func (h *Headway) TileserverServeContainer(ctx context.Context) *dagger.Containe
 		WithDefaultArgs([]string{"/app/configure-and-run.sh"})
 }
 
-// Builds mbtiles using Planetiler
-func (h *Headway) Mbtiles(ctx context.Context) (*dagger.File, error) {
+// Builds maptiles using Planetiler
+func (h *Headway) Pmtiles(ctx context.Context) (*dagger.File, error) {
 
 	if h.OSMExport == nil || h.OSMExport.File == nil {
-		panic("Headway.OSMExport.File must be set to build mbtiles")
+		panic("Headway.OSMExport.File must be set to build pmtiles")
 	}
 
 	container := dag.Container().
-		From("ghcr.io/onthegomap/planetiler:0.7.0")
+		From("ghcr.io/onthegomap/planetiler:0.10.2")
 
 	memoryScript := h.ServiceDir("tilebuilder").File("percent-of-available-memory")
 	memoryBudget, err := container.
@@ -266,30 +266,34 @@ func (h *Headway) Mbtiles(ctx context.Context) (*dagger.File, error) {
 	fixturesUrl := getEnvWithDefault("HEADWAY_PLANETILER_FIXTURES_URL", "https://data.maps.earth/planetiler_fixtures/sources.tar")
 
 	container = container.
+		WithMountedFile("/tmp/planetile-fixture-sources.tar", downloadFile(fixturesUrl)).
 		WithExec([]string{"mkdir", "-p", "/data/sources"}).
-		WithExec([]string{"sh", "-c", "curl --no-progress-meter " + fixturesUrl + " | tar -x --directory /data/sources"}).
+		WithExec([]string{"sh", "-c", "tar -x --directory /data/sources -f /tmp/planetile-fixture-sources.tar"}).
 		WithMountedFile("/data/data.osm.pbf", h.OSMExport.File)
 
 	entrypoint, err := container.Entrypoint(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get entrypoint: %w", err)
 	}
-
+	output := "/data/output.pmtiles"
+	entrypoint = append(entrypoint,
+		"--osm_path=/data/data.osm.pbf",
+		"--force",
+		fmt.Sprintf("--output=%s", output),
+	)
 	if h.IsPlanetBuild {
 		container = container.WithExec(append(entrypoint,
-			"--osm_path=/data/data.osm.pbf",
-			"--force",
 			"--bounds=planet",
 			"--nodemap-type=array",
 			"--storage=mmap",
 			fmt.Sprintf("-Xmx%d", memoryBudget),
 			"-XX:MaxHeapFreeRatio=40",
 		))
 	} else {
-		container = container.WithExec(append(entrypoint, "--osm_path=/data/data.osm.pbf", "--force"))
+		container = container.WithExec(entrypoint)
 	}
 
-	return container.File("/data/output.mbtiles"), nil
+	return container.File(output), nil
 }
 
 /**
@@ -564,6 +568,24 @@ func WithAptPackages(container *dagger.Container, packages ...string) *dagger.Co
 	return container.WithExec([]string{"sh", "-c", cmd})
 }
 
+// Publish a container to multiple registry addresses in a single Dagger
+// session, so every tag points at the same image digest.
+func (h *Headway) PublishMulti(
+	ctx context.Context,
+	container *dagger.Container,
+	addresses []string,
+) ([]string, error) {
+	out := make([]string, 0, len(addresses))
+	for _, addr := range addresses {
+		ref, err := container.Publish(ctx, addr)
+		if err != nil {
+			return nil, fmt.Errorf("publishing %q: %w", addr, err)
+		}
+		out = append(out, ref)
+	}
+	return out, nil
+}
+
 func compressDir(dir *dagger.Directory) *dagger.File {
 	container := slimContainer("zstd").
 		WithExec([]string{"mkdir", "/app"}).

docker-compose-with-transit.yaml

@@ -3,8 +3,8 @@ services:
     image: ghcr.io/headwaymaps/tileserver-init:latest
     env_file: builds/${HEADWAY_AREA:?}/.env
     environment:
-      AREAMAP_ARTIFACT_SOURCE: /bootstrap/${HEADWAY_AREA}.mbtiles
-      AREAMAP_ARTIFACT_DEST: /data/tiles/areamap.mbtiles
+      AREAMAP_ARTIFACT_SOURCE: /bootstrap/${HEADWAY_AREA}.pmtiles
+      AREAMAP_ARTIFACT_DEST: /data/tiles/areamap.pmtiles
       TERRAIN_ARTIFACT_SOURCE: /bootstrap/terrain.mbtiles
       TERRAIN_ARTIFACT_DEST: /data/tiles/terrain.mbtiles
       LANDCOVER_ARTIFACT_SOURCE: /bootstrap/landcover.mbtiles