From 56f1811d69c94fb6dc0ae009c253036432bb6fe3 Mon Sep 17 00:00:00 2001 From: masterashu Date: Wed, 22 Apr 2026 22:59:08 +0530 Subject: [PATCH 1/6] Add go remote debugging support in TILT --- .vscode/launch.json | 23 +++++++++++++++++++++++ Tiltfile | 3 ++- docker-compose-dev.yml | 2 ++ stacks/flow.Dockerfile | 18 +++++++++++++++++- 4 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..c734b42dd --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,23 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Docker Go Debug", + "type": "go", + "request": "attach", + "mode": "remote", + "remotePath": "/root", + "debugAdapter": "dlv-dap", + "hideSystemGoroutines": true, + "showGlobalVariables": true, + "substitutePath": [ + { + "from": "${workspaceFolder}/flow", + "to": "/root/flow" + } + ], + "port": 40000, + "host": "127.0.0.1" + } + ] +} diff --git a/Tiltfile b/Tiltfile index 93ae6fccd..a84fe0948 100644 --- a/Tiltfile +++ b/Tiltfile @@ -16,8 +16,9 @@ docker_build('flow-api', '.', docker_build('flow-worker', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-worker', + target='flow-worker-debug', only=['flow/', 'stacks/flow.Dockerfile'], + build_args={'DEBUG_BUILD':'1'}, ignore=flow_ignore, ) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 339f34072..d4897def0 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -161,6 +161,8 @@ services: context: . dockerfile: stacks/flow.Dockerfile target: flow-worker + ports: + - 40000:40000 environment: <<: [*catalog-config, *flow-worker-env, *minio-config] extra_hosts: diff --git a/stacks/flow.Dockerfile b/stacks/flow.Dockerfile index 040301913..a257f72d2 100644 --- a/stacks/flow.Dockerfile +++ b/stacks/flow.Dockerfile @@ -1,6 +1,10 @@ # syntax=docker/dockerfile:1.23@sha256:2780b5c3bab67f1f76c781860de469442999ed1a0d7992a5efdf2cffc0e3d769 FROM golang:1.26-alpine@sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1 AS builder +# Allow build flags to be passed in at build time, for example debug flags +ARG DEBUG_BUILD +ENV DEBUG_BUILD=${DEBUG_BUILD} + RUN apk add --no-cache gcc geos-dev musl-dev WORKDIR /root/flow @@ -21,7 +25,8 @@ ENV CGO_ENABLED=1 # Generate the typed handler wrapper RUN go generate ENV GOCACHE=/root/.cache/go-build -RUN --mount=type=cache,target="/root/.cache/go-build" go build -o /root/peer-flow +RUN --mount=type=cache,target="/root/.cache/go-build" go build ${DEBUG_BUILD:+-gcflags} ${DEBUG_BUILD:+"all=-N -l"} -o /root/peer-flow +RUN --mount=type=cache,target="/root/.cache/go-build" go install github.com/go-delve/delve/cmd/dlv@latest FROM alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS flow-base ENV TZ=UTC @@ -58,6 +63,17 @@ ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} ENTRYPOINT ["./peer-flow", "snapshot-worker"] +FROM flow-base AS flow-worker-debug +ENV OTEL_METRIC_EXPORT_INTERVAL=10000 +ENV OTEL_EXPORTER_OTLP_COMPRESSION=gzip +ARG PEERDB_VERSION_SHA_SHORT +ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} +USER root +COPY --from=builder /go/bin/dlv /usr/local/bin/dlv +USER peerdb +EXPOSE 40000 +ENTRYPOINT ["dlv", "--headless", "--continue", "--accept-multiclient", "--listen=:40000", "--api-version=2", "exec", "./peer-flow", "--", "worker"] + FROM flow-base AS flow-maintenance ARG PEERDB_VERSION_SHA_SHORT From 05a8624ceaf46d67651597f08dc18b05667efacb Mon Sep 17 00:00:00 2001 From: masterashu Date: Mon, 4 May 2026 16:22:56 +0530 Subject: [PATCH 2/6] =?UTF-8?q?=C2=A0Refactor=20dockerfile=20and=20tiltfil?= =?UTF-8?q?e=20to=20allow=20debug=20all=20go=20container=20in=20docker=20c?= =?UTF-8?q?ompose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot --- .env.example | 7 ++++++- .vscode/launch.json | 40 +++++++++++++++++++++++++++++++++++++-- Tiltfile | 9 +++++---- docker-compose-dev.yml | 10 +++++++++- stacks/flow.Dockerfile | 43 ++++++++++++++++++++++++++++++------------ 5 files changed, 89 insertions(+), 20 deletions(-) diff --git a/.env.example b/.env.example index 92bf76ac9..8b860d573 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,11 @@ PG_USER=postgres PG_PASSWORD=postgres PG_DATABASE=postgres +# Set to 1 for enabling go debugger in dockerfile and Tilt Setup +DOCKER_GO_DEBUG_FLOW_API= +DOCKER_GO_DEBUG_FLOW_WORKER= +DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER= + PEERDB_CATALOG_HOST=host.docker.internal PEERDB_CATALOG_PORT=9901 PEERDB_CATALOG_USER=postgres @@ -44,4 +49,4 @@ AWS_EC2_METADATA_DISABLED="true" MINIO_ROOT_USER=_peerdb_minioadmin MINIO_ROOT_PASSWORD=_peerdb_minioadmin -TZ='UTC' \ No newline at end of file +TZ='UTC' diff --git a/.vscode/launch.json b/.vscode/launch.json index c734b42dd..c9174437c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -2,7 +2,7 @@ "version": "0.2.0", "configurations": [ { - "name": "Docker Go Debug", + "name": "Tilt Debug: Flow Worker", "type": "go", "request": "attach", "mode": "remote", @@ -16,7 +16,43 @@ "to": "/root/flow" } ], - "port": 40000, + "port": 4001, + "host": "127.0.0.1" + }, + { + "name": "Tilt Debug: Flow Snapshot Worker", + "type": "go", + "request": "attach", + "mode": "remote", + "remotePath": "/root", + "debugAdapter": "dlv-dap", + "hideSystemGoroutines": true, + "showGlobalVariables": true, + "substitutePath": [ + { + "from": "${workspaceFolder}/flow", + "to": "/root/flow" + } + ], + "port": 4002, + "host": "127.0.0.1" + }, + { + "name": "Tilt Debug: Flow API", + "type": "go", + "request": "attach", + "mode": "remote", + "remotePath": "/root", + "debugAdapter": "dlv-dap", + "hideSystemGoroutines": true, + "showGlobalVariables": true, + "substitutePath": [ + { + "from": "${workspaceFolder}/flow", + "to": "/root/flow" + } + ], + "port": 4003, "host": "127.0.0.1" } ] diff --git a/Tiltfile b/Tiltfile index a84fe0948..26c3ebb75 100644 --- a/Tiltfile +++ b/Tiltfile @@ -8,7 +8,7 @@ flow_ignore = ['flow/e2e/', 'flow/**/*_test.go'] docker_build('flow-api', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-api', + target='flow-api-debug' if os.getenv('DOCKER_GO_DEBUG_FLOW_API') == '1' else 'flow-api', only=['flow/', 'stacks/flow.Dockerfile'], ignore=flow_ignore, build_args={'PEERDB_VERSION_SHA_SHORT': os.getenv('PEERDB_VERSION_SHA_SHORT', 'unknown')}, @@ -16,16 +16,17 @@ docker_build('flow-api', '.', docker_build('flow-worker', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-worker-debug', + target='flow-worker-debug' if os.getenv('DOCKER_GO_DEBUG_FLOW_WORKER') == '1' else 'flow-worker', only=['flow/', 'stacks/flow.Dockerfile'], - build_args={'DEBUG_BUILD':'1'}, + build_args={'DEBUG_BUILD': os.getenv('DOCKER_GO_DEBUG_FLOW_WORKER','')}, ignore=flow_ignore, ) docker_build('flow-snapshot-worker', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-snapshot-worker', + target='flow-snapshot-worker-debug' if os.getenv('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER') == '1' else 'flow-snapshot-worker', only=['flow/', 'stacks/flow.Dockerfile'], + build_args={'DEBUG_BUILD': os.getenv('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER','')}, ignore=flow_ignore, ) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index d4897def0..4700dbc3c 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -129,6 +129,7 @@ services: - 8112:8112 - 8113:8113 - 5732:5732 + - 4003:40000 environment: <<: [*catalog-config, *flow-worker-env, *minio-config] PEERDB_ALLOWED_TARGETS: @@ -148,6 +149,8 @@ services: context: . dockerfile: stacks/flow.Dockerfile target: flow-snapshot-worker + ports: + - 4002:40000 environment: <<: [*catalog-config, *flow-worker-env, *minio-config] depends_on: @@ -162,7 +165,12 @@ services: dockerfile: stacks/flow.Dockerfile target: flow-worker ports: - - 40000:40000 + - 4001:40000 + # The below options were mentioned to be required, but it works without them, so leaving them commented out for now. If you run into issues with Delve, try uncommenting these. + # security_opt: + # - "seccomp:unconfined" # Required for Delve + # cap_add: + # - SYS_PTRACE # Required for Delve environment: <<: [*catalog-config, *flow-worker-env, *minio-config] extra_hosts: diff --git a/stacks/flow.Dockerfile b/stacks/flow.Dockerfile index a257f72d2..f80bfdddb 100644 --- a/stacks/flow.Dockerfile +++ b/stacks/flow.Dockerfile @@ -37,6 +37,14 @@ RUN apk add --no-cache ca-certificates geos && \ USER peerdb WORKDIR /home/peerdb COPY --from=builder --chown=peerdb /root/peer-flow . +ENTRYPOINT [ "/home/peerdb/peer-flow" ] + +# Debug Image with Delve installed and the binary built with debug flags +FROM flow-base AS flow-base-debug +USER root +COPY --from=builder /go/bin/dlv /usr/local/bin/dlv +EXPOSE 40000 +ENTRYPOINT ["dlv", "--headless", "--continue", "--accept-multiclient", "--listen=:40000", "--api-version=2", "exec", "/home/peerdb/peer-flow", "--"] FROM flow-base AS flow-api @@ -44,7 +52,15 @@ ARG PEERDB_VERSION_SHA_SHORT ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} EXPOSE 8112 8113 -ENTRYPOINT ["./peer-flow", "api", "--port", "8112", "--gateway-port", "8113"] +CMD ["api", "--port", "8112", "--gateway-port", "8113"] + +FROM flow-base-debug AS flow-api-debug + +ARG PEERDB_VERSION_SHA_SHORT +ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} + +EXPOSE 8112 8113 +CMD ["api", "--port", "8112", "--gateway-port", "8113"] FROM flow-base AS flow-worker @@ -54,28 +70,31 @@ ENV OTEL_EXPORTER_OTLP_COMPRESSION=gzip ARG PEERDB_VERSION_SHA_SHORT ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} -ENTRYPOINT ["./peer-flow", "worker"] +CMD ["worker"] + +FROM flow-base-debug AS flow-worker-debug +ENV OTEL_METRIC_EXPORT_INTERVAL=10000 +ENV OTEL_EXPORTER_OTLP_COMPRESSION=gzip +ARG PEERDB_VERSION_SHA_SHORT +ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} +CMD ["./peer-flow", "--", "worker"] + FROM flow-base AS flow-snapshot-worker ARG PEERDB_VERSION_SHA_SHORT ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} -ENTRYPOINT ["./peer-flow", "snapshot-worker"] +CMD ["snapshot-worker"] +FROM flow-base-debug AS flow-snapshot-worker-debug -FROM flow-base AS flow-worker-debug -ENV OTEL_METRIC_EXPORT_INTERVAL=10000 -ENV OTEL_EXPORTER_OTLP_COMPRESSION=gzip ARG PEERDB_VERSION_SHA_SHORT ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} -USER root -COPY --from=builder /go/bin/dlv /usr/local/bin/dlv -USER peerdb -EXPOSE 40000 -ENTRYPOINT ["dlv", "--headless", "--continue", "--accept-multiclient", "--listen=:40000", "--api-version=2", "exec", "./peer-flow", "--", "worker"] +CMD ["snapshot-worker"] + FROM flow-base AS flow-maintenance ARG PEERDB_VERSION_SHA_SHORT ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} -ENTRYPOINT ["./peer-flow", "maintenance"] +CMD ["maintenance"] From 0feb42ffd9d8146d42da3ead425d103bcc23f54d Mon Sep 17 00:00:00 2001 From: masterashu Date: Mon, 4 May 2026 16:55:54 +0530 Subject: [PATCH 3/6] Fix typos --- Tiltfile | 25 ++++++++++++------------- stacks/flow.Dockerfile | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/Tiltfile b/Tiltfile index 26c3ebb75..b69310005 100644 --- a/Tiltfile +++ b/Tiltfile @@ -4,29 +4,35 @@ allow_k8s_contexts(k8s_context()) # to unblock local() in local set-ups with a K docker_compose('./docker-compose-dev.yml') +def resolve_env(var_name, default=None): + for line in str(read_file('.env')).splitlines(): + if line.startswith(var_name + '='): + return line.strip().split('=', 1)[1] + return default + flow_ignore = ['flow/e2e/', 'flow/**/*_test.go'] docker_build('flow-api', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-api-debug' if os.getenv('DOCKER_GO_DEBUG_FLOW_API') == '1' else 'flow-api', + target='flow-api-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_API') == '1' else 'flow-api', only=['flow/', 'stacks/flow.Dockerfile'], ignore=flow_ignore, - build_args={'PEERDB_VERSION_SHA_SHORT': os.getenv('PEERDB_VERSION_SHA_SHORT', 'unknown')}, + build_args={'DEBUG_BUILD': resolve_env('DOCKER_GO_DEBUG_FLOW_API',''),'PEERDB_VERSION_SHA_SHORT': os.getenv('PEERDB_VERSION_SHA_SHORT', 'unknown')}, ) docker_build('flow-worker', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-worker-debug' if os.getenv('DOCKER_GO_DEBUG_FLOW_WORKER') == '1' else 'flow-worker', + target='flow-worker-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_WORKER') == '1' else 'flow-worker', only=['flow/', 'stacks/flow.Dockerfile'], - build_args={'DEBUG_BUILD': os.getenv('DOCKER_GO_DEBUG_FLOW_WORKER','')}, + build_args={'DEBUG_BUILD': resolve_env('DOCKER_GO_DEBUG_FLOW_WORKER','')}, ignore=flow_ignore, ) docker_build('flow-snapshot-worker', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-snapshot-worker-debug' if os.getenv('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER') == '1' else 'flow-snapshot-worker', + target='flow-snapshot-worker-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER') == '1' else 'flow-snapshot-worker', only=['flow/', 'stacks/flow.Dockerfile'], - build_args={'DEBUG_BUILD': os.getenv('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER','')}, + build_args={'DEBUG_BUILD': resolve_env('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER','')}, ignore=flow_ignore, ) @@ -226,13 +232,6 @@ def connector_test(connector, extra_deps=[], vars_overrides={}): ) # These are overrides to provide different MySQL flavors with the same test definitions. - -def resolve_env(var_name): - for line in str(read_file('.env')).splitlines(): - if line.startswith(var_name + '='): - return line.strip().split('=', 1)[1] - return None - mysql_gtid_vars = { 'CI_MYSQL_PORT': resolve_env('CI_MYSQL_GTID_PORT'), 'CI_MYSQL_VERSION': resolve_env('CI_MYSQL_GTID_VERSION'), diff --git a/stacks/flow.Dockerfile b/stacks/flow.Dockerfile index f80bfdddb..9bd9d1e34 100644 --- a/stacks/flow.Dockerfile +++ b/stacks/flow.Dockerfile @@ -77,7 +77,7 @@ ENV OTEL_METRIC_EXPORT_INTERVAL=10000 ENV OTEL_EXPORTER_OTLP_COMPRESSION=gzip ARG PEERDB_VERSION_SHA_SHORT ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} -CMD ["./peer-flow", "--", "worker"] +CMD ["worker"] FROM flow-base AS flow-snapshot-worker From fd2b66f7f2471454fa85f659f5ffdd5ccae4e3ac Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan <39612799+masterashu@users.noreply.github.com> Date: Tue, 5 May 2026 21:10:10 +0530 Subject: [PATCH 4/6] Add TEMPORAL_DEBUG=1 for debug stages --- stacks/flow.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/stacks/flow.Dockerfile b/stacks/flow.Dockerfile index 9bd9d1e34..605ac62a0 100644 --- a/stacks/flow.Dockerfile +++ b/stacks/flow.Dockerfile @@ -43,6 +43,7 @@ ENTRYPOINT [ "/home/peerdb/peer-flow" ] FROM flow-base AS flow-base-debug USER root COPY --from=builder /go/bin/dlv /usr/local/bin/dlv +ENV TEMPORAL_DEBUG=1 EXPOSE 40000 ENTRYPOINT ["dlv", "--headless", "--continue", "--accept-multiclient", "--listen=:40000", "--api-version=2", "exec", "/home/peerdb/peer-flow", "--"] From 20b169f2a1ef2265f7418e81bf83ef300317f911 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan <39612799+masterashu@users.noreply.github.com> Date: Tue, 5 May 2026 21:17:49 +0530 Subject: [PATCH 5/6] add condition to installing dlv --- stacks/flow.Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stacks/flow.Dockerfile b/stacks/flow.Dockerfile index 605ac62a0..05dbcdcf1 100644 --- a/stacks/flow.Dockerfile +++ b/stacks/flow.Dockerfile @@ -26,7 +26,9 @@ ENV CGO_ENABLED=1 RUN go generate ENV GOCACHE=/root/.cache/go-build RUN --mount=type=cache,target="/root/.cache/go-build" go build ${DEBUG_BUILD:+-gcflags} ${DEBUG_BUILD:+"all=-N -l"} -o /root/peer-flow -RUN --mount=type=cache,target="/root/.cache/go-build" go install github.com/go-delve/delve/cmd/dlv@latest +RUN --mount=type=cache,target="/root/.cache/go-build" if [[ "$DEBUG_BUILD" = "1" ]]; then \ + go install github.com/go-delve/delve/cmd/dlv@latest; \ + fi FROM alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS flow-base ENV TZ=UTC From b9281f0afcada4af07f31aadbfb353d3613bd1d6 Mon Sep 17 00:00:00 2001 From: masterashu Date: Wed, 6 May 2026 16:59:38 +0530 Subject: [PATCH 6/6] Add support for customizing port using env, and ability to run multiple instances of tilt for peerdb Co-authored-by: Copilot --- .env.example | 15 +++++++++++ Tiltfile | 27 +++++++++++++------- ancillary-docker-compose.yml | 2 +- docker-compose-dev.yml | 49 ++++++++++++++++++++++++------------ 4 files changed, 67 insertions(+), 26 deletions(-) diff --git a/.env.example b/.env.example index 8b860d573..1295c55f6 100644 --- a/.env.example +++ b/.env.example @@ -50,3 +50,18 @@ MINIO_ROOT_USER=_peerdb_minioadmin MINIO_ROOT_PASSWORD=_peerdb_minioadmin TZ='UTC' + +## Port Overrides for parallel tilt and docker-compose runs. +# PEERDB_CATALOG_PORT=19901 +# PEERDB_NEXUS_PORT=19900 +# PEERDB_SWITCHBOARD_PORT=15732 +# PEERBD_UI_PORT=13030 +# TEMPORAL_PORT=17233 +# TEMPORAL_UI_PORT=18085 +# FLOW_API_PORT=18112 +# FLOW_API_HTTP_PORT=18113 +# DOCKER_GO_DEBUG_PORT_FLOW_WORKER=14001 +# DOCKER_GO_DEBUG_PORT_FLOW_SNAPSHOT_WORKER=14002 +# DOCKER_GO_DEBUG_PORT_FLOW_API=14003 +# MINIO_PORT=19001 +# MINIO_CONSOLE_PORT=19002 diff --git a/Tiltfile b/Tiltfile index b69310005..db095270f 100644 --- a/Tiltfile +++ b/Tiltfile @@ -2,19 +2,28 @@ update_settings(max_parallel_updates=10) allow_k8s_contexts(k8s_context()) # to unblock local() in local set-ups with a Kubernetes context configured, like Docker Desktop -docker_compose('./docker-compose-dev.yml') - def resolve_env(var_name, default=None): for line in str(read_file('.env')).splitlines(): if line.startswith(var_name + '='): return line.strip().split('=', 1)[1] return default +docker_compose('./docker-compose-dev.yml', project_name='peerdb-' + resolve_env('DEFAULT_TILT_PORT', '10350'), env_file='.env') + +peerbd_ui_port = resolve_env('PEERBD_UI_PORT', '3030') +temporal_port = resolve_env('TEMPORAL_PORT', '7233') +temporal_ui_port = resolve_env('TEMPORAL_UI_PORT', '8085') +flow_api_grpc_port = resolve_env('FLOW_API_PORT', '8112') +flow_api_http_port = resolve_env('FLOW_API_HTTP_PORT', '8113') +docker_go_debug_port_flow_worker = resolve_env('DOCKER_GO_DEBUG_PORT_FLOW_WORKER', '4001') +docker_go_debug_port_flow_snapshot_worker = resolve_env('DOCKER_GO_DEBUG_PORT_FLOW_SNAPSHOT_WORKER', '4002') +docker_go_debug_port_flow_api = resolve_env('DOCKER_GO_DEBUG_PORT_FLOW_API', '4003') + flow_ignore = ['flow/e2e/', 'flow/**/*_test.go'] docker_build('flow-api', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-api-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_API') == '1' else 'flow-api', + target='flow-api-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_API') in ('1', 'true') else 'flow-api', only=['flow/', 'stacks/flow.Dockerfile'], ignore=flow_ignore, build_args={'DEBUG_BUILD': resolve_env('DOCKER_GO_DEBUG_FLOW_API',''),'PEERDB_VERSION_SHA_SHORT': os.getenv('PEERDB_VERSION_SHA_SHORT', 'unknown')}, @@ -22,7 +31,7 @@ docker_build('flow-api', '.', docker_build('flow-worker', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-worker-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_WORKER') == '1' else 'flow-worker', + target='flow-worker-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_WORKER') in ('1', 'true') else 'flow-worker', only=['flow/', 'stacks/flow.Dockerfile'], build_args={'DEBUG_BUILD': resolve_env('DOCKER_GO_DEBUG_FLOW_WORKER','')}, ignore=flow_ignore, @@ -30,7 +39,7 @@ docker_build('flow-worker', '.', docker_build('flow-snapshot-worker', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-snapshot-worker-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER') == '1' else 'flow-snapshot-worker', + target='flow-snapshot-worker-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER') in ('1', 'true') else 'flow-snapshot-worker', only=['flow/', 'stacks/flow.Dockerfile'], build_args={'DEBUG_BUILD': resolve_env('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER','')}, ignore=flow_ignore, @@ -59,14 +68,14 @@ local_resource( ) dc_resource('peerdb-ui', resource_deps=['proto-gen'], labels=['PeerDB'], links=[ - link('http://localhost:3030', 'PeerDB UI'), + link('http://localhost:' + str(peerbd_ui_port), 'PeerDB UI'), ]) dc_resource('flow-api', resource_deps=['proto-gen'], labels=['PeerDB'], links=[ - link('http://localhost:8112', 'Flow API gRPC'), - link('http://localhost:8113', 'Flow API HTTP'), + link('http://localhost:' + str(flow_api_grpc_port), 'Flow API gRPC'), + link('http://localhost:' + str(flow_api_http_port), 'Flow API HTTP'), ]) dc_resource('temporal-ui', labels=['PeerDB'], links=[ - link('http://localhost:8085', 'Temporal UI'), + link('http://localhost:' + str(temporal_ui_port), 'Temporal UI'), ]) dc_resource('catalog', labels=['PeerDB']) dc_resource('temporal', labels=['PeerDB']) diff --git a/ancillary-docker-compose.yml b/ancillary-docker-compose.yml index 8787032cd..1b9225050 100755 --- a/ancillary-docker-compose.yml +++ b/ancillary-docker-compose.yml @@ -193,5 +193,5 @@ volumes: networks: default: - name: peerdb_network + name: peerdb_network_${DEFAULT_TILT_PORT:-default} external: true diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 4700dbc3c..f4bb1ff34 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,12 +1,29 @@ name: peerdb-quickstart-dev +# Public facing ports for services to prevent conflicts. +# Overridable using environment variables for flexibility when using multiple tilt environments +x-ports: + - &peerdb-catalog-port ${PEERDB_CATALOG_PORT:-9901}:5432 + - &peerdb-nexus-port ${PEERDB_NEXUS_PORT:-9900}:9900 + - &peerdb-switchboard-port ${PEERDB_SWITCHBOARD_PORT:-5732}:5732 + - &peerdb-ui-port ${PEERBD_UI_PORT:-3030}:3000 + - &temporal-port ${TEMPORAL_PORT:-7233}:7233 + - &temporal-ui-port ${TEMPORAL_UI_PORT:-8085}:8080 + - &flow-api-grpc-port ${FLOW_API_PORT:-8112}:8112 + - &flow-api-http-port ${FLOW_API_HTTP_PORT:-8113}:8113 + - &flow-worker-debug-port ${DOCKER_GO_DEBUG_PORT_FLOW_WORKER:-4001}:40000 + - &flow-snapshot-worker-debug-port ${DOCKER_GO_DEBUG_PORT_FLOW_SNAPSHOT_WORKER:-4002}:40000 + - &flow-api-debug-port ${DOCKER_GO_DEBUG_PORT_FLOW_API:-4003}:40000 + - &minio-port ${MINIO_PORT:-9001}:9000 + - &minio-console-port ${MINIO_CONSOLE_PORT:-9002}:36987 + x-minio-config: &minio-config PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: _peerdb_minioadmin MINIO_ROOT_USER: _peerdb_minioadmin PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: _peerdb_minioadmin MINIO_ROOT_PASSWORD: _peerdb_minioadmin PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1 - PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://host.docker.internal:9001 + PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://host.docker.internal:${MINIO_PORT:-9001} PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdb x-catalog-config: &catalog-config @@ -44,7 +61,7 @@ services: image: postgres:18-alpine@sha256:54451ecb8ab38c24c3ec123f2fd501303a3a1856a5c66e98cecf2460d5e1e9d7 command: -c config_file=/etc/postgresql.conf ports: - - 9901:5432 + - *peerdb-catalog-port environment: PGUSER: postgres POSTGRES_PASSWORD: postgres @@ -53,7 +70,7 @@ services: extra_hosts: - "host.docker.internal:host-gateway" volumes: - - pgdata:/var/lib/postgresql/data + - pgdata:/var/lib/postgresql - ./volumes/postgresql.conf:/etc/postgresql.conf - ./volumes/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d healthcheck: @@ -77,7 +94,7 @@ services: - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml image: temporalio/auto-setup:1.29@sha256:d33d1402e8b895329e27a0d0e7cc2f034e0aa70f0dab6aa95b306d7ae2716f67 ports: - - 7233:7233 + - *temporal-port volumes: - ./volumes/temporal-dynamicconfig:/etc/temporal/config/dynamicconfig labels: @@ -114,7 +131,7 @@ services: - TEMPORAL_CSRF_COOKIE_INSECURE=true image: temporalio/ui:2.49.1@sha256:a066bdf5c4de689cabaf80cc357871f1db5e6d750a6bcfc42e877b913e31ef24 ports: - - 8085:8080 + - *temporal-ui-port flow-api: container_name: flow_api @@ -126,10 +143,10 @@ services: args: PEERDB_VERSION_SHA_SHORT: ${PEERDB_VERSION_SHA_SHORT:-} ports: - - 8112:8112 - - 8113:8113 - - 5732:5732 - - 4003:40000 + - *flow-api-grpc-port + - *flow-api-http-port + - *peerdb-switchboard-port + - *flow-api-debug-port environment: <<: [*catalog-config, *flow-worker-env, *minio-config] PEERDB_ALLOWED_TARGETS: @@ -150,7 +167,7 @@ services: dockerfile: stacks/flow.Dockerfile target: flow-snapshot-worker ports: - - 4002:40000 + - *flow-snapshot-worker-debug-port environment: <<: [*catalog-config, *flow-worker-env, *minio-config] depends_on: @@ -165,7 +182,7 @@ services: dockerfile: stacks/flow.Dockerfile target: flow-worker ports: - - 4001:40000 + - *flow-worker-debug-port # The below options were mentioned to be required, but it works without them, so leaving them commented out for now. If you run into issues with Delve, try uncommenting these. # security_opt: # - "seccomp:unconfined" # Required for Delve @@ -196,7 +213,7 @@ services: RUST_LOG: info RUST_BACKTRACE: 1 ports: - - 9900:9900 + - *peerdb-nexus-port depends_on: catalog: condition: service_healthy @@ -209,7 +226,7 @@ services: dockerfile: stacks/peerdb-ui.Dockerfile target: dev ports: - - 3030:3000 + - *peerdb-ui-port volumes: # Mount local ui/ directory for hot reload - ./ui:/app @@ -239,8 +256,8 @@ services: volumes: - minio-data:/data ports: - - "9001:9000" - - "9002:36987" + - *minio-port + - *minio-console-port environment: <<: *minio-config entrypoint: > @@ -258,4 +275,4 @@ volumes: networks: default: - name: peerdb_network + name: peerdb_network_${DEFAULT_TILT_PORT:-default}