Cross service trace propagation

README.md

@@ -265,6 +265,7 @@ rake app=ruby/rails7-delayed-job mode=collector app:up
 rake app=ruby/rails7-goodjob mode=collector app:up
 rake app=ruby/rails7-postgres mode=collector app:up
 rake app=ruby/rails7-sequel mode=collector app:up
+rake app=ruby/rails7-sidekiq mode=collector app:up
 rake app=ruby/rails7-solid-cache mode=collector app:up
 rake app=ruby/rails7-solid-queue mode=collector app:up
 rake app=ruby/rails8-delayed-job mode=collector app:up

python/django4-asgi/app/appsignal_python_opentelemetry/settings.py

@@ -26,7 +26,7 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
-ALLOWED_HOSTS = ['app', 'localhost']
+ALLOWED_HOSTS = ['app', 'downstream', 'localhost']
 
 
 # Application definition

python/django4-asgi/app/appsignal_python_opentelemetry/views.py

@@ -1,3 +1,4 @@
+import os
 import requests
 import json
 import time
@@ -84,7 +85,7 @@ def error_queue_inline(request):
 
 
 def make_request(request):
-    requests.get('https://www.appsignal.com/')
+    requests.get(os.environ.get("DOWNSTREAM_URL", "https://www.appsignal.com/"))
     return HttpResponse("I did a request to appsignal.com!")
 
 

python/django4-asgi/app/processmon.downstream.toml

@@ -0,0 +1,14 @@
+[[paths_to_watch]]
+path = "/app"
+
+# Downstream web server on :4002 (no celery here; that is the worker split).
+[processes.django]
+command = "python3"
+args = [
+  "-m",
+  "uvicorn",
+  "appsignal_python_opentelemetry.asgi:application",
+  "--host",
+  "0.0.0.0"
+]
+working_dir = "/app"

python/django4-asgi/app/processmon.toml

@@ -11,13 +11,3 @@ args = [
   "0.0.0.0"
 ]
 working_dir = "/app"
-
-[processes.celery]
-command = "celery"
-args = [
-  "--app=tasks",
-  "worker",
-  "--loglevel=INFO",
-  "--queues=high-priority,low-priority"
-]
-working_dir = "/app"

python/django4-asgi/app/processmon.worker.toml

@@ -0,0 +1,15 @@
+[[paths_to_watch]]
+path = "/app"
+
+# The worker service runs only the background worker. Work enqueued by the web
+# `app` service is performed here, so it reports under the worker's own app name
+# (agent mode) or `worker` service name (collector mode).
+[processes.celery]
+command = "celery"
+args = [
+  "--app=tasks",
+  "worker",
+  "--loglevel=INFO",
+  "--queues=high-priority,low-priority"
+]
+working_dir = "/app"

python/django4-asgi/commands/run-downstream

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -eu
+
+cd /app
+
+# The `app` service builds the appsignal dist into the shared /integration
+# volume and runs migrations, then serves HTTP. Wait for it to be reachable:
+# that means the dist is built (so we install from it without racing the app's
+# `hatch build`) and the schema is migrated (so we never migrate from here).
+echo "Waiting for the app service to be ready..."
+until python3 -c "import socket; socket.create_connection(('app', 4001), 2).close()" >/dev/null 2>&1; do
+  echo "App not ready yet, retrying..."
+  sleep 2
+done
+
+echo "Installing dependencies"
+pip3 install -r requirements.txt
+pip3 install /integration/dist/* --force-reinstall
+
+export UVICORN_PORT=4002
+
+echo "Starting downstream processes"
+/commands/processmon processmon.downstream.toml

python/django4-asgi/commands/run-worker

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -eu
+
+cd /app
+
+# The `app` service builds the appsignal dist into the shared /integration
+# volume and (for the Django apps) runs migrations, then serves HTTP. Wait for
+# it to be reachable before installing here: that means the dist is built (so we
+# install from it without racing the app's `hatch build`) and the schema is
+# migrated (so this container never migrates).
+echo "Waiting for the app service to be ready..."
+until python3 -c "import socket; socket.create_connection(('app', 4001), 2).close()" >/dev/null 2>&1; do
+  echo "App not ready yet, retrying..."
+  sleep 2
+done
+
+echo "Installing dependencies"
+pip3 install -r requirements.txt
+pip3 install /integration/dist/* --force-reinstall
+
+echo "Starting worker processes"
+/commands/processmon processmon.worker.toml

python/django4-asgi/docker-compose.agent.yml

@@ -5,3 +5,9 @@ services:
   app:
     environment:
       - APPSIGNAL_APP_NAME=python/django4-asgi
+  downstream:
+    environment:
+      - APPSIGNAL_APP_NAME=python/django4-asgi-downstream
+  worker:
+    environment:
+      - APPSIGNAL_APP_NAME=python/django4-asgi-worker

python/django4-asgi/docker-compose.collector.yml

@@ -6,4 +6,15 @@ services:
   app:
     environment:
       - APPSIGNAL_APP_NAME=python/django4-asgi-collector
+      - APPSIGNAL_SERVICE_NAME=app
+      - APPSIGNAL_COLLECTOR_ENDPOINT=http://appsignal-collector:8099
+  downstream:
+    environment:
+      - APPSIGNAL_APP_NAME=python/django4-asgi-collector
+      - APPSIGNAL_SERVICE_NAME=downstream
+      - APPSIGNAL_COLLECTOR_ENDPOINT=http://appsignal-collector:8099
+  worker:
+    environment:
+      - APPSIGNAL_APP_NAME=python/django4-asgi-collector
+      - APPSIGNAL_SERVICE_NAME=worker
       - APPSIGNAL_COLLECTOR_ENDPOINT=http://appsignal-collector:8099

python/django4-asgi/docker-compose.shared.yml

@@ -28,6 +28,36 @@ services:
       - ../../appsignal_key.env
     environment:
       - PORT=4001
+      - DOWNSTREAM_URL=http://downstream:4002/
+    volumes:
+      - ./app:/app
+      - ../integration:/integration
+  # Second instance of the same image (image-only; only `app` builds). Receives
+  # the HTTP request the `app` service makes, so the trace continues into it.
+  downstream:
+    image: python/django4-asgi
+    command: /commands/run-downstream
+    depends_on:
+      - postgres
+      - redis
+    env_file:
+      - postgres.env
+      - ../../appsignal.env
+      - ../../appsignal_key.env
+    volumes:
+      - ./app:/app
+      - ../integration:/integration
+  # Runs only the background worker (image-only; only `app` builds).
+  worker:
+    image: python/django4-asgi
+    command: /commands/run-worker
+    depends_on:
+      - postgres
+      - redis
+    env_file:
+      - postgres.env
+      - ../../appsignal.env
+      - ../../appsignal_key.env
     volumes:
       - ./app:/app
       - ../integration:/integration

python/django4-celery/app/appsignal_python_opentelemetry/settings.py

@@ -26,7 +26,7 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
-ALLOWED_HOSTS = ['app', 'localhost']
+ALLOWED_HOSTS = ['app', 'downstream', 'localhost']
 
 
 # Application definition

python/django4-celery/app/appsignal_python_opentelemetry/views.py

@@ -1,3 +1,4 @@
+import os
 import requests
 import json
 import time
@@ -91,9 +92,13 @@ def error_queue_inline(request):
     return HttpResponse("I ran an error_task inline!")
 
 
+# Calls a second, separately-instrumented app (the `downstream` service) so the
+# injected traceparent is extracted there and the trace spans both apps. Falls
+# back to the external, uninstrumented URL when DOWNSTREAM_URL is unset, so the
+# setup still works as a single service.
 def make_request(request):
-    requests.get('https://www.appsignal.com/')
-    return HttpResponse("I did a request to appsignal.com!")
+    requests.get(os.environ.get("DOWNSTREAM_URL", "https://www.appsignal.com/"))
+    return HttpResponse("I did a request downstream!")
 
 
 def custom_instrumentation(request):

python/django4-celery/app/processmon.downstream.toml

@@ -0,0 +1,15 @@
+[[paths_to_watch]]
+path = "/app"
+
+# The downstream service is a second instance of the same Django app on a
+# different port. It receives the HTTP request the `app` service makes, so the
+# trace continues into it. In collector mode it reports under the `downstream`
+# service name; in agent mode under a separate `-downstream` app.
+[processes.django]
+command = "python3"
+args = [
+  "manage.py",
+  "runserver",
+  "0.0.0.0:4002"
+]
+working_dir = "/app"

python/django4-celery/app/processmon.toml

@@ -9,13 +9,3 @@ args = [
   "0.0.0.0:4001"
 ]
 working_dir = "/app"
-
-[processes.celery]
-command = "celery"
-args = [
-  "--app=tasks",
-  "worker",
-  "--loglevel=INFO",
-  "--queues=high-priority,low-priority"
-]
-working_dir = "/app"

python/django4-celery/app/processmon.worker.toml

@@ -0,0 +1,15 @@
+[[paths_to_watch]]
+path = "/app"
+
+# The worker service runs only the background worker. Work enqueued by the web
+# `app` service is performed here, so it reports under the worker's own app name
+# (agent mode) or `worker` service name (collector mode).
+[processes.celery]
+command = "celery"
+args = [
+  "--app=tasks",
+  "worker",
+  "--loglevel=INFO",
+  "--queues=high-priority,low-priority"
+]
+working_dir = "/app"

python/django4-celery/commands/run-downstream

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -eu
+
+cd /app
+
+# The `app` service owns building the appsignal dist into the shared
+# /integration volume and running migrations. Wait for it to start serving
+# before doing anything here. That single signal means the dist is built, so we
+# can install from it without racing the app's `hatch build`, and it means the
+# schema is migrated, so this container never runs migrations itself.
+echo "Waiting for the app service to be ready..."
+until python3 -c "import socket; socket.create_connection(('app', 4001), 2).close()" >/dev/null 2>&1; do
+  echo "App not ready yet, retrying..."
+  sleep 2
+done
+
+echo "Installing dependencies"
+pip3 install -r requirements.txt
+pip3 install /integration/dist/* --force-reinstall
+
+echo "Starting downstream processes"
+/commands/processmon processmon.downstream.toml

python/django4-celery/commands/run-worker

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -eu
+
+cd /app
+
+# The `app` service builds the appsignal dist into the shared /integration
+# volume and (for the Django apps) runs migrations, then serves HTTP. Wait for
+# it to be reachable before installing here: that means the dist is built (so we
+# install from it without racing the app's `hatch build`) and the schema is
+# migrated (so this container never migrates).
+echo "Waiting for the app service to be ready..."
+until python3 -c "import socket; socket.create_connection(('app', 4001), 2).close()" >/dev/null 2>&1; do
+  echo "App not ready yet, retrying..."
+  sleep 2
+done
+
+echo "Installing dependencies"
+pip3 install -r requirements.txt
+pip3 install /integration/dist/* --force-reinstall
+
+echo "Starting worker processes"
+/commands/processmon processmon.worker.toml

python/django4-celery/docker-compose.agent.yml

@@ -5,3 +5,9 @@ services:
   app:
     environment:
       - APPSIGNAL_APP_NAME=python/django4-celery
+  downstream:
+    environment:
+      - APPSIGNAL_APP_NAME=python/django4-celery-downstream
+  worker:
+    environment:
+      - APPSIGNAL_APP_NAME=python/django4-celery-worker

python/django4-celery/docker-compose.collector.yml

@@ -6,4 +6,15 @@ services:
   app:
     environment:
       - APPSIGNAL_APP_NAME=python/django4-celery-collector
+      - APPSIGNAL_SERVICE_NAME=app
+      - APPSIGNAL_COLLECTOR_ENDPOINT=http://appsignal-collector:8099
+  downstream:
+    environment:
+      - APPSIGNAL_APP_NAME=python/django4-celery-collector
+      - APPSIGNAL_SERVICE_NAME=downstream
+      - APPSIGNAL_COLLECTOR_ENDPOINT=http://appsignal-collector:8099
+  worker:
+    environment:
+      - APPSIGNAL_APP_NAME=python/django4-celery-collector
+      - APPSIGNAL_SERVICE_NAME=worker
       - APPSIGNAL_COLLECTOR_ENDPOINT=http://appsignal-collector:8099

python/django4-celery/docker-compose.shared.yml

@@ -28,6 +28,41 @@ services:
       - ../../appsignal_key.env
     environment:
       - PORT=4001
+      # The make_request view calls this second, separately-instrumented app so
+      # the trace propagates across services. Falls back to an external URL when
+      # unset (single-service runs).
+      - DOWNSTREAM_URL=http://downstream:4002/
+    volumes:
+      - ./app:/app
+      - ../integration:/integration
+  # A second instance of the same image, running as a distinct AppSignal app
+  # (agent mode) or service (collector mode). It receives the HTTP request the
+  # `app` service makes, so the trace continues into it. Image-only: only `app`
+  # builds, so `docker compose build` does not collide on the shared image tag.
+  downstream:
+    image: python/django4-celery
+    command: /commands/run-downstream
+    depends_on:
+      - postgres
+      - redis
+    env_file:
+      - postgres.env
+      - ../../appsignal.env
+      - ../../appsignal_key.env
+    volumes:
+      - ./app:/app
+      - ../integration:/integration
+  # Runs only the background worker (image-only; only `app` builds).
+  worker:
+    image: python/django4-celery
+    command: /commands/run-worker
+    depends_on:
+      - postgres
+      - redis
+    env_file:
+      - postgres.env
+      - ../../appsignal.env
+      - ../../appsignal_key.env
     volumes:
       - ./app:/app
       - ../integration:/integration

python/django4-wsgi/app/appsignal_python_opentelemetry/settings.py

@@ -26,7 +26,7 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
-ALLOWED_HOSTS = ['app', 'localhost']
+ALLOWED_HOSTS = ['app', 'downstream', 'localhost']
 
 
 # Application definition