Add go remote debugging support in TILT (#4216)

masterashu · Copilot · web-flow · commit ded6df198d9b · 2026-05-08T18:33:48.000+05:30
Summary of Changes:
* Added go compiler flags to flow build command in flow.dockerfile
(disabled optimizations, recommended for debugging)
* Added a new stage in flow.dockerfile with dlv (delve debugger)
entrypoint which exposes endpoint to attach debugger to and updated the
same in Tiltfile
* VSCode launch config which attaches the debugger and resolved
breakpoints etc.

---------

Co-authored-by: Copilot &lt;copilot@github.com&gt;
diff --git a/.env.example b/.env.example
@@ -50,4 +50,24 @@ AWS_EC2_METADATA_DISABLED="true"
 MINIO_ROOT_USER=_peerdb_minioadmin
 MINIO_ROOT_PASSWORD=_peerdb_minioadmin
 
-TZ='UTC'
+# Set to 1 for enabling go debugger in dockerfile and Tilt Setup
+DOCKER_GO_DEBUG_FLOW_API=
+DOCKER_GO_DEBUG_FLOW_WORKER=
+DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER=
+
+TZ='UTC'
+
+## Port Overrides for parallel tilt and docker-compose runs.
+# PEERDB_CATALOG_PORT=19901
+# PEERDB_NEXUS_PORT=19900
+# PEERDB_SWITCHBOARD_PORT=15732
+# PEERBD_UI_PORT=13030
+# TEMPORAL_PORT=17233
+# TEMPORAL_UI_PORT=18085
+# FLOW_API_PORT=18112
+# FLOW_API_HTTP_PORT=18113
+# DOCKER_GO_DEBUG_PORT_FLOW_WORKER=14001
+# DOCKER_GO_DEBUG_PORT_FLOW_SNAPSHOT_WORKER=14002
+# DOCKER_GO_DEBUG_PORT_FLOW_API=14003
+# MINIO_PORT=19001
+# MINIO_CONSOLE_PORT=19002
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -0,0 +1,59 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Tilt Debug: Flow Worker",
+      "type": "go",
+      "request": "attach",
+      "mode": "remote",
+      "remotePath": "/root",
+      "debugAdapter": "dlv-dap",
+      "hideSystemGoroutines": true,
+      "showGlobalVariables": true,
+      "substitutePath": [
+        {
+          "from": "${workspaceFolder}/flow",
+          "to": "/root/flow"
+        }
+      ],
+      "port": 4001,
+      "host": "127.0.0.1"
+    },
+    {
+      "name": "Tilt Debug: Flow Snapshot Worker",
+      "type": "go",
+      "request": "attach",
+      "mode": "remote",
+      "remotePath": "/root",
+      "debugAdapter": "dlv-dap",
+      "hideSystemGoroutines": true,
+      "showGlobalVariables": true,
+      "substitutePath": [
+        {
+          "from": "${workspaceFolder}/flow",
+          "to": "/root/flow"
+        }
+      ],
+      "port": 4002,
+      "host": "127.0.0.1"
+    },
+    {
+      "name": "Tilt Debug: Flow API",
+      "type": "go",
+      "request": "attach",
+      "mode": "remote",
+      "remotePath": "/root",
+      "debugAdapter": "dlv-dap",
+      "hideSystemGoroutines": true,
+      "showGlobalVariables": true,
+      "substitutePath": [
+        {
+          "from": "${workspaceFolder}/flow",
+          "to": "/root/flow"
+        }
+      ],
+      "port": 4003,
+      "host": "127.0.0.1"
+    }
+  ]
+}
diff --git a/Tiltfile b/Tiltfile
@@ -2,29 +2,46 @@ update_settings(max_parallel_updates=10)
 
 allow_k8s_contexts(k8s_context()) # to unblock local() in local set-ups with a Kubernetes context configured, like Docker Desktop
 
-docker_compose('./docker-compose-dev.yml')
+def resolve_env(var_name, default=None):
+    for line in str(read_file('.env')).splitlines():
+        if line.startswith(var_name + '='):
+            return line.strip().split('=', 1)[1]
+    return default
+
+docker_compose('./docker-compose-dev.yml', project_name='peerdb-' + resolve_env('DEFAULT_TILT_PORT', '10350'), env_file='.env')
+
+peerbd_ui_port = resolve_env('PEERBD_UI_PORT', '3030')
+temporal_port = resolve_env('TEMPORAL_PORT', '7233')
+temporal_ui_port = resolve_env('TEMPORAL_UI_PORT', '8085')
+flow_api_grpc_port = resolve_env('FLOW_API_PORT', '8112')
+flow_api_http_port = resolve_env('FLOW_API_HTTP_PORT', '8113')
+docker_go_debug_port_flow_worker = resolve_env('DOCKER_GO_DEBUG_PORT_FLOW_WORKER', '4001')
+docker_go_debug_port_flow_snapshot_worker = resolve_env('DOCKER_GO_DEBUG_PORT_FLOW_SNAPSHOT_WORKER', '4002')
+docker_go_debug_port_flow_api = resolve_env('DOCKER_GO_DEBUG_PORT_FLOW_API', '4003')
 
 flow_ignore = ['flow/e2e/', 'flow/**/*_test.go']
 
 docker_build('flow-api', '.',
     dockerfile='stacks/flow.Dockerfile',
-    target='flow-api',
+    target='flow-api-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_API') in ('1', 'true') else 'flow-api',
     only=['flow/', 'stacks/flow.Dockerfile'],
     ignore=flow_ignore,
-    build_args={'PEERDB_VERSION_SHA_SHORT': os.getenv('PEERDB_VERSION_SHA_SHORT', 'unknown')},
+    build_args={'DEBUG_BUILD': resolve_env('DOCKER_GO_DEBUG_FLOW_API',''),'PEERDB_VERSION_SHA_SHORT': os.getenv('PEERDB_VERSION_SHA_SHORT', 'unknown')},
 )
 
 docker_build('flow-worker', '.',
     dockerfile='stacks/flow.Dockerfile',
-    target='flow-worker',
+    target='flow-worker-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_WORKER') in ('1', 'true') else 'flow-worker',
     only=['flow/', 'stacks/flow.Dockerfile'],
+    build_args={'DEBUG_BUILD': resolve_env('DOCKER_GO_DEBUG_FLOW_WORKER','')},
     ignore=flow_ignore,
 )
 
 docker_build('flow-snapshot-worker', '.',
     dockerfile='stacks/flow.Dockerfile',
-    target='flow-snapshot-worker',
+    target='flow-snapshot-worker-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER') in ('1', 'true') else 'flow-snapshot-worker',
     only=['flow/', 'stacks/flow.Dockerfile'],
+    build_args={'DEBUG_BUILD': resolve_env('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER','')},
     ignore=flow_ignore,
 )
 
@@ -51,14 +68,14 @@ local_resource(
 )
 
 dc_resource('peerdb-ui', resource_deps=['proto-gen'], labels=['PeerDB'], links=[
-    link('http://localhost:3030', 'PeerDB UI'),
+    link('http://localhost:' + str(peerbd_ui_port), 'PeerDB UI'),
 ])
 dc_resource('flow-api', resource_deps=['proto-gen'], labels=['PeerDB'], links=[
-    link('http://localhost:8112', 'Flow API gRPC'),
-    link('http://localhost:8113', 'Flow API HTTP'),
+    link('http://localhost:' + str(flow_api_grpc_port), 'Flow API gRPC'),
+    link('http://localhost:' + str(flow_api_http_port), 'Flow API HTTP'),
 ])
 dc_resource('temporal-ui', labels=['PeerDB'], links=[
-    link('http://localhost:8085', 'Temporal UI'),
+    link('http://localhost:' + str(temporal_ui_port), 'Temporal UI'),
 ])
 dc_resource('catalog', labels=['PeerDB'])
 dc_resource('temporal', labels=['PeerDB'])
@@ -235,13 +252,6 @@ def connector_test(connector, extra_deps=[], vars_overrides={}):
     )
 
 # These are overrides to provide different MySQL flavors with the same test definitions.
-
-def resolve_env(var_name):
-    for line in str(read_file('.env')).splitlines():
-        if line.startswith(var_name + '='):
-            return line.strip().split('=', 1)[1]
-    return None
-
 mysql_gtid_vars = {
     'CI_MYSQL_PORT': resolve_env('CI_MYSQL_GTID_PORT'),
     'CI_MYSQL_VERSION': resolve_env('CI_MYSQL_GTID_VERSION'),
diff --git a/ancillary-docker-compose.yml b/ancillary-docker-compose.yml
@@ -215,5 +215,5 @@ volumes:
 
 networks:
   default:
-    name: peerdb_network
+    name: peerdb_network_${DEFAULT_TILT_PORT:-default}
     external: true
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
@@ -1,12 +1,29 @@
 name: peerdb-quickstart-dev
 
+# Public facing ports for services to prevent conflicts.
+# Overridable using environment variables for flexibility when using multiple tilt environments
+x-ports:
+  - &peerdb-catalog-port      ${PEERDB_CATALOG_PORT:-9901}:5432
+  - &peerdb-nexus-port        ${PEERDB_NEXUS_PORT:-9900}:9900
+  - &peerdb-switchboard-port  ${PEERDB_SWITCHBOARD_PORT:-5732}:5732
+  - &peerdb-ui-port           ${PEERBD_UI_PORT:-3030}:3000
+  - &temporal-port            ${TEMPORAL_PORT:-7233}:7233
+  - &temporal-ui-port         ${TEMPORAL_UI_PORT:-8085}:8080
+  - &flow-api-grpc-port            ${FLOW_API_PORT:-8112}:8112
+  - &flow-api-http-port       ${FLOW_API_HTTP_PORT:-8113}:8113
+  - &flow-worker-debug-port   ${DOCKER_GO_DEBUG_PORT_FLOW_WORKER:-4001}:40000
+  - &flow-snapshot-worker-debug-port ${DOCKER_GO_DEBUG_PORT_FLOW_SNAPSHOT_WORKER:-4002}:40000
+  - &flow-api-debug-port      ${DOCKER_GO_DEBUG_PORT_FLOW_API:-4003}:40000
+  - &minio-port               ${MINIO_PORT:-9001}:9000
+  - &minio-console-port       ${MINIO_CONSOLE_PORT:-9002}:36987
+
 x-minio-config: &minio-config
   PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: _peerdb_minioadmin
   MINIO_ROOT_USER: _peerdb_minioadmin
   PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: _peerdb_minioadmin
   MINIO_ROOT_PASSWORD: _peerdb_minioadmin
   PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1
-  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://host.docker.internal:9001
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://host.docker.internal:${MINIO_PORT:-9001}
   PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdb
 
 x-catalog-config: &catalog-config
@@ -44,7 +61,7 @@ services:
     image: postgres:18-alpine@sha256:54451ecb8ab38c24c3ec123f2fd501303a3a1856a5c66e98cecf2460d5e1e9d7
     command: -c config_file=/etc/postgresql.conf
     ports:
-      - 9901:5432
+      - *peerdb-catalog-port
     environment:
       PGUSER: postgres
       POSTGRES_PASSWORD: postgres
@@ -77,7 +94,7 @@ services:
       - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml
     image: temporalio/auto-setup:1.29@sha256:d33d1402e8b895329e27a0d0e7cc2f034e0aa70f0dab6aa95b306d7ae2716f67
     ports:
-      - 7233:7233
+      - *temporal-port
     volumes:
       - ./volumes/temporal-dynamicconfig:/etc/temporal/config/dynamicconfig
     labels:
@@ -114,7 +131,7 @@ services:
       - TEMPORAL_CSRF_COOKIE_INSECURE=true
     image: temporalio/ui:2.49.1@sha256:a066bdf5c4de689cabaf80cc357871f1db5e6d750a6bcfc42e877b913e31ef24
     ports:
-      - 8085:8080
+      - *temporal-ui-port
 
   flow-api:
     container_name: flow_api
@@ -126,9 +143,10 @@ services:
       args:
         PEERDB_VERSION_SHA_SHORT: ${PEERDB_VERSION_SHA_SHORT:-}
     ports:
-      - 8112:8112
-      - 8113:8113
-      - 5732:5732
+      - *flow-api-grpc-port
+      - *flow-api-http-port
+      - *peerdb-switchboard-port
+      - *flow-api-debug-port
     environment:
       <<: [*catalog-config, *flow-worker-env, *minio-config]
       PEERDB_ALLOWED_TARGETS:
@@ -148,6 +166,8 @@ services:
       context: .
       dockerfile: stacks/flow.Dockerfile
       target: flow-snapshot-worker
+    ports:
+      - *flow-snapshot-worker-debug-port
     environment:
       <<: [*catalog-config, *flow-worker-env, *minio-config]
     depends_on:
@@ -161,6 +181,13 @@ services:
       context: .
       dockerfile: stacks/flow.Dockerfile
       target: flow-worker
+    ports:
+      - *flow-worker-debug-port
+    # The below options were mentioned to be required, but it works without them, so leaving them commented out for now. If you run into issues with Delve, try uncommenting these.
+    # security_opt:
+    #   - "seccomp:unconfined"  # Required for Delve
+    # cap_add:
+    #   - SYS_PTRACE         # Required for Delve
     environment:
       <<: [*catalog-config, *flow-worker-env, *minio-config]
     extra_hosts:
@@ -186,7 +213,7 @@ services:
       RUST_LOG: info
       RUST_BACKTRACE: 1
     ports:
-      - 9900:9900
+      - *peerdb-nexus-port
     depends_on:
       catalog:
         condition: service_healthy
@@ -199,7 +226,7 @@ services:
       dockerfile: stacks/peerdb-ui.Dockerfile
       target: dev
     ports:
-      - 3030:3000
+      - *peerdb-ui-port
     volumes:
       # Mount local ui/ directory for hot reload
       - ./ui:/app
@@ -229,8 +256,8 @@ services:
     volumes:
       - minio-data:/data
     ports:
-      - "9001:9000"
-      - "9002:36987"
+      - *minio-port
+      - *minio-console-port
     environment:
       <<: *minio-config
     entrypoint: >
@@ -248,4 +275,4 @@ volumes:
 
 networks:
   default:
-    name: peerdb_network
+    name: peerdb_network_${DEFAULT_TILT_PORT:-default}
diff --git a/stacks/flow.Dockerfile b/stacks/flow.Dockerfile
@@ -1,6 +1,10 @@
 # syntax=docker/dockerfile:1.23@sha256:2780b5c3bab67f1f76c781860de469442999ed1a0d7992a5efdf2cffc0e3d769
 
 FROM golang:1.26-alpine@sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1 AS builder
+# Allow build flags to be passed in at build time, for example debug flags
+ARG DEBUG_BUILD
+ENV DEBUG_BUILD=${DEBUG_BUILD}
+
 RUN apk add --no-cache gcc geos-dev musl-dev
 WORKDIR /root/flow
 
@@ -21,7 +25,10 @@ ENV CGO_ENABLED=1
 # Generate the typed handler wrapper
 RUN go generate
 ENV GOCACHE=/root/.cache/go-build
-RUN --mount=type=cache,target="/root/.cache/go-build" go build -o /root/peer-flow
+RUN --mount=type=cache,target="/root/.cache/go-build" go build ${DEBUG_BUILD:+-gcflags} ${DEBUG_BUILD:+"all=-N -l"} -o /root/peer-flow
+RUN --mount=type=cache,target="/root/.cache/go-build" if [[ "$DEBUG_BUILD" = "1" ]]; then \
+    go install github.com/go-delve/delve/cmd/dlv@latest; \
+  fi
 
 FROM alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS flow-base
 ENV TZ=UTC
@@ -32,14 +39,31 @@ RUN apk add --no-cache ca-certificates geos && \
 USER peerdb
 WORKDIR /home/peerdb
 COPY --from=builder --chown=peerdb /root/peer-flow .
+ENTRYPOINT [ "/home/peerdb/peer-flow" ]
+
+# Debug Image with Delve installed and the binary built with debug flags
+FROM flow-base AS flow-base-debug
+USER root
+COPY --from=builder /go/bin/dlv /usr/local/bin/dlv
+ENV TEMPORAL_DEBUG=1
+EXPOSE 40000
+ENTRYPOINT ["dlv", "--headless", "--continue", "--accept-multiclient", "--listen=:40000", "--api-version=2", "exec", "/home/peerdb/peer-flow", "--"]
 
 FROM flow-base AS flow-api
 
 ARG PEERDB_VERSION_SHA_SHORT
 ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT}
 
 EXPOSE 8112 8113
-ENTRYPOINT ["./peer-flow", "api", "--port", "8112", "--gateway-port", "8113"]
+CMD ["api", "--port", "8112", "--gateway-port", "8113"]
+
+FROM flow-base-debug AS flow-api-debug
+
+ARG PEERDB_VERSION_SHA_SHORT
+ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT}
+
+EXPOSE 8112 8113
+CMD ["api", "--port", "8112", "--gateway-port", "8113"]
 
 FROM flow-base AS flow-worker
 
@@ -53,17 +77,31 @@ ENV OTEL_EXPORTER_OTLP_COMPRESSION=gzip
 ARG PEERDB_VERSION_SHA_SHORT
 ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT}
 
-ENTRYPOINT ["./peer-flow", "worker"]
+CMD ["worker"]
+
+FROM flow-base-debug AS flow-worker-debug
+ENV OTEL_METRIC_EXPORT_INTERVAL=10000
+ENV OTEL_EXPORTER_OTLP_COMPRESSION=gzip
+ARG PEERDB_VERSION_SHA_SHORT
+ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT}
+CMD ["worker"]
+
 
 FROM flow-base AS flow-snapshot-worker
 
 ARG PEERDB_VERSION_SHA_SHORT
 ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT}
-ENTRYPOINT ["./peer-flow", "snapshot-worker"]
+CMD ["snapshot-worker"]
+
+FROM flow-base-debug AS flow-snapshot-worker-debug
+
+ARG PEERDB_VERSION_SHA_SHORT
+ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT}
+CMD ["snapshot-worker"]
 
 
 FROM flow-base AS flow-maintenance
 
 ARG PEERDB_VERSION_SHA_SHORT
 ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT}
-ENTRYPOINT ["./peer-flow", "maintenance"]
+CMD ["maintenance"]
