diff --git a/.env.example b/.env.example index 58d5df8c1..681c4b011 100644 --- a/.env.example +++ b/.env.example @@ -50,4 +50,24 @@ AWS_EC2_METADATA_DISABLED="true" MINIO_ROOT_USER=_peerdb_minioadmin MINIO_ROOT_PASSWORD=_peerdb_minioadmin -TZ='UTC' \ No newline at end of file +# Set to 1 for enabling go debugger in dockerfile and Tilt Setup +DOCKER_GO_DEBUG_FLOW_API= +DOCKER_GO_DEBUG_FLOW_WORKER= +DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER= + +TZ='UTC' + +## Port Overrides for parallel tilt and docker-compose runs. +# PEERDB_CATALOG_PORT=19901 +# PEERDB_NEXUS_PORT=19900 +# PEERDB_SWITCHBOARD_PORT=15732 +# PEERBD_UI_PORT=13030 +# TEMPORAL_PORT=17233 +# TEMPORAL_UI_PORT=18085 +# FLOW_API_PORT=18112 +# FLOW_API_HTTP_PORT=18113 +# DOCKER_GO_DEBUG_PORT_FLOW_WORKER=14001 +# DOCKER_GO_DEBUG_PORT_FLOW_SNAPSHOT_WORKER=14002 +# DOCKER_GO_DEBUG_PORT_FLOW_API=14003 +# MINIO_PORT=19001 +# MINIO_CONSOLE_PORT=19002 diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..c9174437c --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,59 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Tilt Debug: Flow Worker", + "type": "go", + "request": "attach", + "mode": "remote", + "remotePath": "/root", + "debugAdapter": "dlv-dap", + "hideSystemGoroutines": true, + "showGlobalVariables": true, + "substitutePath": [ + { + "from": "${workspaceFolder}/flow", + "to": "/root/flow" + } + ], + "port": 4001, + "host": "127.0.0.1" + }, + { + "name": "Tilt Debug: Flow Snapshot Worker", + "type": "go", + "request": "attach", + "mode": "remote", + "remotePath": "/root", + "debugAdapter": "dlv-dap", + "hideSystemGoroutines": true, + "showGlobalVariables": true, + "substitutePath": [ + { + "from": "${workspaceFolder}/flow", + "to": "/root/flow" + } + ], + "port": 4002, + "host": "127.0.0.1" + }, + { + "name": "Tilt Debug: Flow API", + "type": "go", + "request": "attach", + "mode": "remote", + "remotePath": "/root", + "debugAdapter": "dlv-dap", + "hideSystemGoroutines": true, + "showGlobalVariables": true, + "substitutePath": [ + { + "from": "${workspaceFolder}/flow", + "to": "/root/flow" + } + ], + "port": 4003, + "host": "127.0.0.1" + } + ] +} diff --git a/Tiltfile b/Tiltfile index 8b92b3b50..1158b9a77 100644 --- a/Tiltfile +++ b/Tiltfile @@ -2,29 +2,46 @@ update_settings(max_parallel_updates=10) allow_k8s_contexts(k8s_context()) # to unblock local() in local set-ups with a Kubernetes context configured, like Docker Desktop -docker_compose('./docker-compose-dev.yml') +def resolve_env(var_name, default=None): + for line in str(read_file('.env')).splitlines(): + if line.startswith(var_name + '='): + return line.strip().split('=', 1)[1] + return default + +docker_compose('./docker-compose-dev.yml', project_name='peerdb-' + resolve_env('DEFAULT_TILT_PORT', '10350'), env_file='.env') + +peerbd_ui_port = resolve_env('PEERBD_UI_PORT', '3030') +temporal_port = resolve_env('TEMPORAL_PORT', '7233') +temporal_ui_port = resolve_env('TEMPORAL_UI_PORT', '8085') +flow_api_grpc_port = resolve_env('FLOW_API_PORT', '8112') +flow_api_http_port = resolve_env('FLOW_API_HTTP_PORT', '8113') +docker_go_debug_port_flow_worker = resolve_env('DOCKER_GO_DEBUG_PORT_FLOW_WORKER', '4001') +docker_go_debug_port_flow_snapshot_worker = resolve_env('DOCKER_GO_DEBUG_PORT_FLOW_SNAPSHOT_WORKER', '4002') +docker_go_debug_port_flow_api = resolve_env('DOCKER_GO_DEBUG_PORT_FLOW_API', '4003') flow_ignore = ['flow/e2e/', 'flow/**/*_test.go'] docker_build('flow-api', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-api', + target='flow-api-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_API') in ('1', 'true') else 'flow-api', only=['flow/', 'stacks/flow.Dockerfile'], ignore=flow_ignore, - build_args={'PEERDB_VERSION_SHA_SHORT': os.getenv('PEERDB_VERSION_SHA_SHORT', 'unknown')}, + build_args={'DEBUG_BUILD': resolve_env('DOCKER_GO_DEBUG_FLOW_API',''),'PEERDB_VERSION_SHA_SHORT': os.getenv('PEERDB_VERSION_SHA_SHORT', 'unknown')}, ) docker_build('flow-worker', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-worker', + target='flow-worker-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_WORKER') in ('1', 'true') else 'flow-worker', only=['flow/', 'stacks/flow.Dockerfile'], + build_args={'DEBUG_BUILD': resolve_env('DOCKER_GO_DEBUG_FLOW_WORKER','')}, ignore=flow_ignore, ) docker_build('flow-snapshot-worker', '.', dockerfile='stacks/flow.Dockerfile', - target='flow-snapshot-worker', + target='flow-snapshot-worker-debug' if resolve_env('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER') in ('1', 'true') else 'flow-snapshot-worker', only=['flow/', 'stacks/flow.Dockerfile'], + build_args={'DEBUG_BUILD': resolve_env('DOCKER_GO_DEBUG_FLOW_SNAPSHOT_WORKER','')}, ignore=flow_ignore, ) @@ -51,14 +68,14 @@ local_resource( ) dc_resource('peerdb-ui', resource_deps=['proto-gen'], labels=['PeerDB'], links=[ - link('http://localhost:3030', 'PeerDB UI'), + link('http://localhost:' + str(peerbd_ui_port), 'PeerDB UI'), ]) dc_resource('flow-api', resource_deps=['proto-gen'], labels=['PeerDB'], links=[ - link('http://localhost:8112', 'Flow API gRPC'), - link('http://localhost:8113', 'Flow API HTTP'), + link('http://localhost:' + str(flow_api_grpc_port), 'Flow API gRPC'), + link('http://localhost:' + str(flow_api_http_port), 'Flow API HTTP'), ]) dc_resource('temporal-ui', labels=['PeerDB'], links=[ - link('http://localhost:8085', 'Temporal UI'), + link('http://localhost:' + str(temporal_ui_port), 'Temporal UI'), ]) dc_resource('catalog', labels=['PeerDB']) dc_resource('temporal', labels=['PeerDB']) @@ -235,13 +252,6 @@ def connector_test(connector, extra_deps=[], vars_overrides={}): ) # These are overrides to provide different MySQL flavors with the same test definitions. - -def resolve_env(var_name): - for line in str(read_file('.env')).splitlines(): - if line.startswith(var_name + '='): - return line.strip().split('=', 1)[1] - return None - mysql_gtid_vars = { 'CI_MYSQL_PORT': resolve_env('CI_MYSQL_GTID_PORT'), 'CI_MYSQL_VERSION': resolve_env('CI_MYSQL_GTID_VERSION'), diff --git a/ancillary-docker-compose.yml b/ancillary-docker-compose.yml index b252bf15a..38702db5c 100755 --- a/ancillary-docker-compose.yml +++ b/ancillary-docker-compose.yml @@ -215,5 +215,5 @@ volumes: networks: default: - name: peerdb_network + name: peerdb_network_${DEFAULT_TILT_PORT:-default} external: true diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 0e035bd11..f4bb1ff34 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,12 +1,29 @@ name: peerdb-quickstart-dev +# Public facing ports for services to prevent conflicts. +# Overridable using environment variables for flexibility when using multiple tilt environments +x-ports: + - &peerdb-catalog-port ${PEERDB_CATALOG_PORT:-9901}:5432 + - &peerdb-nexus-port ${PEERDB_NEXUS_PORT:-9900}:9900 + - &peerdb-switchboard-port ${PEERDB_SWITCHBOARD_PORT:-5732}:5732 + - &peerdb-ui-port ${PEERBD_UI_PORT:-3030}:3000 + - &temporal-port ${TEMPORAL_PORT:-7233}:7233 + - &temporal-ui-port ${TEMPORAL_UI_PORT:-8085}:8080 + - &flow-api-grpc-port ${FLOW_API_PORT:-8112}:8112 + - &flow-api-http-port ${FLOW_API_HTTP_PORT:-8113}:8113 + - &flow-worker-debug-port ${DOCKER_GO_DEBUG_PORT_FLOW_WORKER:-4001}:40000 + - &flow-snapshot-worker-debug-port ${DOCKER_GO_DEBUG_PORT_FLOW_SNAPSHOT_WORKER:-4002}:40000 + - &flow-api-debug-port ${DOCKER_GO_DEBUG_PORT_FLOW_API:-4003}:40000 + - &minio-port ${MINIO_PORT:-9001}:9000 + - &minio-console-port ${MINIO_CONSOLE_PORT:-9002}:36987 + x-minio-config: &minio-config PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: _peerdb_minioadmin MINIO_ROOT_USER: _peerdb_minioadmin PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: _peerdb_minioadmin MINIO_ROOT_PASSWORD: _peerdb_minioadmin PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1 - PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://host.docker.internal:9001 + PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://host.docker.internal:${MINIO_PORT:-9001} PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdb x-catalog-config: &catalog-config @@ -44,7 +61,7 @@ services: image: postgres:18-alpine@sha256:54451ecb8ab38c24c3ec123f2fd501303a3a1856a5c66e98cecf2460d5e1e9d7 command: -c config_file=/etc/postgresql.conf ports: - - 9901:5432 + - *peerdb-catalog-port environment: PGUSER: postgres POSTGRES_PASSWORD: postgres @@ -77,7 +94,7 @@ services: - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml image: temporalio/auto-setup:1.29@sha256:d33d1402e8b895329e27a0d0e7cc2f034e0aa70f0dab6aa95b306d7ae2716f67 ports: - - 7233:7233 + - *temporal-port volumes: - ./volumes/temporal-dynamicconfig:/etc/temporal/config/dynamicconfig labels: @@ -114,7 +131,7 @@ services: - TEMPORAL_CSRF_COOKIE_INSECURE=true image: temporalio/ui:2.49.1@sha256:a066bdf5c4de689cabaf80cc357871f1db5e6d750a6bcfc42e877b913e31ef24 ports: - - 8085:8080 + - *temporal-ui-port flow-api: container_name: flow_api @@ -126,9 +143,10 @@ services: args: PEERDB_VERSION_SHA_SHORT: ${PEERDB_VERSION_SHA_SHORT:-} ports: - - 8112:8112 - - 8113:8113 - - 5732:5732 + - *flow-api-grpc-port + - *flow-api-http-port + - *peerdb-switchboard-port + - *flow-api-debug-port environment: <<: [*catalog-config, *flow-worker-env, *minio-config] PEERDB_ALLOWED_TARGETS: @@ -148,6 +166,8 @@ services: context: . dockerfile: stacks/flow.Dockerfile target: flow-snapshot-worker + ports: + - *flow-snapshot-worker-debug-port environment: <<: [*catalog-config, *flow-worker-env, *minio-config] depends_on: @@ -161,6 +181,13 @@ services: context: . dockerfile: stacks/flow.Dockerfile target: flow-worker + ports: + - *flow-worker-debug-port + # The below options were mentioned to be required, but it works without them, so leaving them commented out for now. If you run into issues with Delve, try uncommenting these. + # security_opt: + # - "seccomp:unconfined" # Required for Delve + # cap_add: + # - SYS_PTRACE # Required for Delve environment: <<: [*catalog-config, *flow-worker-env, *minio-config] extra_hosts: @@ -186,7 +213,7 @@ services: RUST_LOG: info RUST_BACKTRACE: 1 ports: - - 9900:9900 + - *peerdb-nexus-port depends_on: catalog: condition: service_healthy @@ -199,7 +226,7 @@ services: dockerfile: stacks/peerdb-ui.Dockerfile target: dev ports: - - 3030:3000 + - *peerdb-ui-port volumes: # Mount local ui/ directory for hot reload - ./ui:/app @@ -229,8 +256,8 @@ services: volumes: - minio-data:/data ports: - - "9001:9000" - - "9002:36987" + - *minio-port + - *minio-console-port environment: <<: *minio-config entrypoint: > @@ -248,4 +275,4 @@ volumes: networks: default: - name: peerdb_network + name: peerdb_network_${DEFAULT_TILT_PORT:-default} diff --git a/stacks/flow.Dockerfile b/stacks/flow.Dockerfile index 7c2b4dbb2..95c6f6a60 100644 --- a/stacks/flow.Dockerfile +++ b/stacks/flow.Dockerfile @@ -1,6 +1,10 @@ # syntax=docker/dockerfile:1.23@sha256:2780b5c3bab67f1f76c781860de469442999ed1a0d7992a5efdf2cffc0e3d769 FROM golang:1.26-alpine@sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1 AS builder +# Allow build flags to be passed in at build time, for example debug flags +ARG DEBUG_BUILD +ENV DEBUG_BUILD=${DEBUG_BUILD} + RUN apk add --no-cache gcc geos-dev musl-dev WORKDIR /root/flow @@ -21,7 +25,10 @@ ENV CGO_ENABLED=1 # Generate the typed handler wrapper RUN go generate ENV GOCACHE=/root/.cache/go-build -RUN --mount=type=cache,target="/root/.cache/go-build" go build -o /root/peer-flow +RUN --mount=type=cache,target="/root/.cache/go-build" go build ${DEBUG_BUILD:+-gcflags} ${DEBUG_BUILD:+"all=-N -l"} -o /root/peer-flow +RUN --mount=type=cache,target="/root/.cache/go-build" if [[ "$DEBUG_BUILD" = "1" ]]; then \ + go install github.com/go-delve/delve/cmd/dlv@latest; \ + fi FROM alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS flow-base ENV TZ=UTC @@ -32,6 +39,15 @@ RUN apk add --no-cache ca-certificates geos && \ USER peerdb WORKDIR /home/peerdb COPY --from=builder --chown=peerdb /root/peer-flow . +ENTRYPOINT [ "/home/peerdb/peer-flow" ] + +# Debug Image with Delve installed and the binary built with debug flags +FROM flow-base AS flow-base-debug +USER root +COPY --from=builder /go/bin/dlv /usr/local/bin/dlv +ENV TEMPORAL_DEBUG=1 +EXPOSE 40000 +ENTRYPOINT ["dlv", "--headless", "--continue", "--accept-multiclient", "--listen=:40000", "--api-version=2", "exec", "/home/peerdb/peer-flow", "--"] FROM flow-base AS flow-api @@ -39,7 +55,15 @@ ARG PEERDB_VERSION_SHA_SHORT ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} EXPOSE 8112 8113 -ENTRYPOINT ["./peer-flow", "api", "--port", "8112", "--gateway-port", "8113"] +CMD ["api", "--port", "8112", "--gateway-port", "8113"] + +FROM flow-base-debug AS flow-api-debug + +ARG PEERDB_VERSION_SHA_SHORT +ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} + +EXPOSE 8112 8113 +CMD ["api", "--port", "8112", "--gateway-port", "8113"] FROM flow-base AS flow-worker @@ -53,17 +77,31 @@ ENV OTEL_EXPORTER_OTLP_COMPRESSION=gzip ARG PEERDB_VERSION_SHA_SHORT ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} -ENTRYPOINT ["./peer-flow", "worker"] +CMD ["worker"] + +FROM flow-base-debug AS flow-worker-debug +ENV OTEL_METRIC_EXPORT_INTERVAL=10000 +ENV OTEL_EXPORTER_OTLP_COMPRESSION=gzip +ARG PEERDB_VERSION_SHA_SHORT +ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} +CMD ["worker"] + FROM flow-base AS flow-snapshot-worker ARG PEERDB_VERSION_SHA_SHORT ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} -ENTRYPOINT ["./peer-flow", "snapshot-worker"] +CMD ["snapshot-worker"] + +FROM flow-base-debug AS flow-snapshot-worker-debug + +ARG PEERDB_VERSION_SHA_SHORT +ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} +CMD ["snapshot-worker"] FROM flow-base AS flow-maintenance ARG PEERDB_VERSION_SHA_SHORT ENV PEERDB_VERSION_SHA_SHORT=${PEERDB_VERSION_SHA_SHORT} -ENTRYPOINT ["./peer-flow", "maintenance"] +CMD ["maintenance"]