docker: jemalloc in published relay image + TLS-terminated stats stack (#480)

Author: gmarzot

.github/workflows/ci-main.yml

@@ -662,43 +662,16 @@ jobs:
       - name: Log in to GHCR
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
-      - name: Write .env
-        working-directory: docker
-        run: |
-          cat > .env <<EOF
-          DOMAIN=${DOMAIN}
-          CERTBOT_EMAIL=gmarzot@openmoq.org
-          MOQX_PORT=${RELAY_PORT}
-          MOQX_ADMIN_PORT=${ADMIN_PORT}
-          MOQX_LOG_LEVEL=0
-          MOQX_VERBOSE=0
-          EOF
-          sed -i 's/^[[:space:]]*//' .env
-
-      - name: Pull and deploy
-        working-directory: docker
-        run: |
-          docker compose pull
-          docker compose down --remove-orphans 2>/dev/null || true
-          # Remove any stale containers with matching names (e.g. from manual docker run)
-          docker rm -f moqx logmon 2>/dev/null || true
-          docker compose up -d
-
-          echo "==> Waiting for admin endpoint..."
-          for i in $(seq 1 30); do
-            if curl -sf http://127.0.0.1:${ADMIN_PORT}/info >/dev/null 2>&1; then
-              break
-            fi
-            sleep 1
-            if [ "$i" -eq 30 ]; then
-              echo "::error::Admin endpoint did not respond within 30s"
-              docker compose logs moqx
-              exit 1
-            fi
-          done
-
-          RESP=$(curl -sf http://127.0.0.1:8000/info)
-          echo "==> Relay deployed: $RESP"
+      # Auto-deploy uses the shared core (docker/relay-deploy.sh) with stats ON,
+      # so main keeps the stats stack + public read-only dashboard live. Pulls
+      # the compose-pinned :latest (this run's just-published image).
+      - name: Deploy (relay + stats + public dashboard)
+        env:
+          ENABLE_STATS: "true"
+          STATS_USER: ${{ secrets.STATS_USER }}
+          STATS_PASSWORD: ${{ secrets.STATS_PASSWORD }}
+          GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+        run: bash docker/relay-deploy.sh
 
   # ════════════════════════════════════════════════════════════════════════════
   # Notify: aggregate status (runs after everything)

.github/workflows/deploy-relay.yml

@@ -1,7 +1,7 @@
 name: deploy relay
 
 # Ops controls for the CI relay (moqx-main.ci.openmoq.org + moqx-000 alias).
-# Restart, redeploy with a specific image tag, or change verbose level.
+# Restart, redeploy with a specific image tag, or change the log level.
 
 on:
   workflow_dispatch:
@@ -12,25 +12,32 @@ on:
         type: boolean
         default: false
       image_tag:
-        description: "Docker image tag (default: derived from branch — main→latest, release/X→X)"
+        description: "Image tag (default derived from branch)"
         required: false
         default: ""
         type: string
       domain:
-        description: "DNS hostname (default: derived from branch — main→moqx-main, release/X→moqx-X)"
+        description: "Hostname (default derived from branch)"
         required: false
         default: ""
         type: string
-      verbose:
-        description: "GLOG verbose level (0=off, 1-3=increasing detail)"
+      logging:
+        description: "Log level (default INFO)"
         required: false
-        default: "0"
+        default: "default"
         type: choice
         options:
-          - "0"
-          - "1"
-          - "2"
-          - "3"
+          - "default"
+          - "WARN"
+          - "DBG1"
+          - "DBG2"
+          - "DBG3"
+          - "DBG4"
+      enable_stats:
+        description: "Enable stats dashboard (default enabled)"
+        required: false
+        type: boolean
+        default: true
 
 permissions:
   contents: read
@@ -65,6 +72,11 @@ jobs:
 
           DOMAIN="${{ inputs.domain }}"
           DOMAIN="${DOMAIN:-moqx-${LABEL}.ci.openmoq.org}"
+          # Qualify a bare label (no dot) into the CI zone, so an explicit
+          # `domain=moqx-main` works the same as the full FQDN.
+          if [[ "$DOMAIN" != *.* ]]; then
+            DOMAIN="${DOMAIN}.ci.openmoq.org"
+          fi
 
           echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT"
           echo "domain=$DOMAIN" >> "$GITHUB_OUTPUT"
@@ -126,73 +138,52 @@ jobs:
         working-directory: docker
         env:
           RESTART_ONLY: ${{ inputs.restart_only }}
-          VERBOSE: ${{ inputs.verbose }}
+          LOGGING: ${{ inputs.logging }}
           DOMAIN: ${{ steps.target.outputs.domain }}
           IMAGE_TAG: ${{ steps.target.outputs.image_tag }}
           AWS_ACCESS_KEY_ID: ${{ secrets.OMOQ_CERTBOT_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.OMOQ_CERTBOT_SECRET_ACCESS_KEY }}
+          ENABLE_STATS: ${{ inputs.enable_stats }}
+          STATS_USER: ${{ secrets.STATS_USER }}
+          STATS_PASSWORD: ${{ secrets.STATS_PASSWORD }}
+          GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
         run: |
           if [ "$RESTART_ONLY" = "true" ]; then
             echo "==> Restarting relay..."
             docker compose restart moqx
+            for i in $(seq 1 30); do
+              curl -sf "http://127.0.0.1:${ADMIN_PORT}/info" >/dev/null 2>&1 && break
+              sleep 1
+              [ "$i" -eq 30 ] && { echo "::error::relay did not respond after restart"; docker compose logs moqx; exit 1; }
+            done
+            echo "==> Relay restarted: $(curl -sf http://127.0.0.1:${ADMIN_PORT}/info)"
+            exit 0
+          fi
+
+          # folly XLOG override ("default" from the dialog = no override).
+          LOGGING_CFG=""
+          [ "${LOGGING}" != "default" ] && LOGGING_CFG="${LOGGING}"
+
+          # Ensure TLS cert
+          CERT="/etc/letsencrypt/live/${DOMAIN}/fullchain.pem"
+          if [ ! -f "$CERT" ]; then
+            echo "::warning::No cert for ${DOMAIN} — provisioning via Route53"
+            sudo -E certbot certonly --dns-route53 \
+              -d "$DOMAIN" --non-interactive --agree-tos \
+              --email gmarzot@openmoq.org
+          elif ! sudo openssl x509 -in "$CERT" -noout -checkend 2592000 2>/dev/null; then
+            echo "::warning::Cert for ${DOMAIN} expires within 30 days — renewing"
+            sudo -E certbot renew --cert-name "$DOMAIN"
           else
-            # Write .env
-            cat > .env <<EOF
-            DOMAIN=${DOMAIN}
-            CERTBOT_EMAIL=gmarzot@openmoq.org
-            MOQX_PORT=${RELAY_PORT}
-            MOQX_ADMIN_PORT=${ADMIN_PORT}
-            MOQX_LOG_LEVEL=0
-            MOQX_VERBOSE=${VERBOSE}
-          EOF
-            sed -i 's/^[[:space:]]*//' .env
-
-            # Ensure TLS cert
-            CERT="/etc/letsencrypt/live/${DOMAIN}/fullchain.pem"
-            if [ ! -f "$CERT" ]; then
-              echo "::warning::No cert for ${DOMAIN} — provisioning via Route53"
-              sudo -E certbot certonly --dns-route53 \
-                -d "$DOMAIN" --non-interactive --agree-tos \
-                --email gmarzot@openmoq.org
-            elif ! sudo openssl x509 -in "$CERT" -noout -checkend 2592000 2>/dev/null; then
-              echo "::warning::Cert for ${DOMAIN} expires within 30 days — renewing"
-              sudo -E certbot renew --cert-name "$DOMAIN"
-            else
-              echo "Cert for ${DOMAIN} is valid"
-            fi
-
-            # Pull and deploy
-            docker pull "ghcr.io/${{ github.repository }}:${IMAGE_TAG}"
-
-            IMAGE="ghcr.io/${{ github.repository }}:${IMAGE_TAG}"
-            if [ "$IMAGE_TAG" != "latest" ]; then
-              docker tag "$IMAGE" "ghcr.io/${{ github.repository }}:latest"
-            fi
-
-            echo "==> Stopping existing container (if any)..."
-            docker compose down --remove-orphans 2>/dev/null || true
-            docker rm -f moqx logmon 2>/dev/null || true
-
-            echo "==> Starting relay on ${DOMAIN}:${RELAY_PORT}..."
-            docker compose up -d
+            echo "Cert for ${DOMAIN} is valid"
           fi
 
-          # Health check (both paths)
-          echo "==> Waiting for admin endpoint..."
-          for i in $(seq 1 30); do
-            if curl -sf http://127.0.0.1:${ADMIN_PORT}/info >/dev/null 2>&1; then
-              break
-            fi
-            sleep 1
-            if [ "$i" -eq 30 ]; then
-              echo "::error::Admin endpoint did not respond within 30s"
-              docker compose logs moqx
-              exit 1
-            fi
-          done
-
-          RESP=$(curl -sf http://127.0.0.1:${ADMIN_PORT}/info)
-          echo "==> Relay running: $RESP"
+          # Deploy via the shared core (same script ci-main uses): writes .env,
+          # pulls IMAGE_TAG (retagged :latest for compose), brings up the relay
+          # + stats + public dashboard when ENABLE_STATS, health check, publish.
+          PULL_IMAGE="ghcr.io/${{ github.repository }}:${IMAGE_TAG}" \
+          MOQX_LOGGING="$LOGGING_CFG" \
+          bash relay-deploy.sh
 
       - name: Notify Slack
         if: always()

.gitignore

@@ -38,3 +38,4 @@ docker/cloudflare.ini
 
 # gh-pages dashboard (lives on gh-pages branch, not main)
 /gh-pages/
+docker/grafana/provisioning/dashboards/archive/

.gitmodules

@@ -2,7 +2,10 @@
 	path = deps/moxygen
 	url = git@github.com:openmoq/moxygen.git
 	branch = main
-	ignore = untracked
+	# "all": pointer drift never shows in status/diff, so bulk git add can't
+	# sweep an accidental pin bump into a feature commit. Intentional bumps
+	# (sync PRs) still stage with an explicit `git add deps/moxygen`.
+	ignore = all
 [submodule "deps/catapult"]
 	path = deps/catapult
 	url = https://github.com/Quicr/catapult.git

docker/.env.example

@@ -10,18 +10,47 @@ MOQX_PORT=4433
 # Admin HTTP port (localhost only)
 MOQX_ADMIN_PORT=8000
 
-# Logging: 0=INFO 1=WARNING 2=ERROR 3=FATAL
-MOQX_LOG_LEVEL=0
+# Log level for the whole stack via folly XLOG (empty = baseline INFO).
+# e.g. DBG2, WARN, or a folly config like INFO,quic=WARN. moqx promotes this to
+# folly's FOLLY_LOGGING env var internally. See docs/logging.md.
+# MOQX_LOGGING=
 
-# Verbose/debug level: 0=off, 1-4=increasing detail
-# MOQX_VERBOSE=0
+# jemalloc allocator for the relay (~10% speedup). Default: auto (enabled).
+# Set to off/false/0 to use the system allocator, or a path to force a lib.
+# MOQX_JEMALLOC=auto
 
 # Dozzle log viewer port (localhost only)
 # MOQX_LOG_PORT=9999
 
 # Per-module verbose logging (e.g. MoQSession=4,MoQForwarder=4)
 # GLOG_vmodule=
 
+# --- Stats stack (compose profile: stats) ---
+# nginx terminates TLS (reusing the relay's Let's Encrypt cert for ${DOMAIN})
+# and reverse-proxies Grafana, Prometheus, the admin endpoints, and logs behind
+# HTTP basic auth. Bring it up with:  docker compose --profile stats up -d
+#
+# Basic-auth credentials for the proxied screens (required for the stats profile)
+STATS_USER=admin
+STATS_PASSWORD=change-me
+# Host bind address for the stats frontend. Default 127.0.0.1 = tunnel-only
+# (reach via `ssh -L 8443:localhost:443 <host>`); set 0.0.0.0 to expose publicly.
+# STATS_BIND=127.0.0.1
+# Published HTTPS / HTTP ports for the stats frontend
+# STATS_HTTPS_PORT=443
+# STATS_HTTP_PORT=80
+# Grafana admin login (Grafana's own console)
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=change-me
+# Prometheus retention: time window and a hard on-disk size cap (whichever hits
+# first wins). Relay metrics are fixed-cardinality, so 3s scrape × 365d is a few
+# GB/yr — comfortably under the 10G cap.
+# PROMETHEUS_RETENTION=365d
+# PROMETHEUS_RETENTION_SIZE=10GB
+# TSDB backing store. Default = named volume (survives redeploys, lives under
+# /var/lib/docker). Set an absolute host path to pin it on a dedicated disk.
+# PROMETHEUS_DATA_DIR=prometheus-data
+
 # --- Route53 credentials (certbot-route53 profile only) ---
 # AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
 # AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY

docker/Dockerfile

@@ -32,6 +32,7 @@ FROM debian:bookworm-slim
 RUN apt-get update && apt-get install -y --no-install-recommends \
         libunwind8 libsodium23 libboost-context1.74.0 \
         libgoogle-glog0v6 libgflags2.2 libdouble-conversion3 \
+        libjemalloc2 \
         curl gettext-base \
     && rm -rf /var/lib/apt/lists/*