Skip to content

Commit 6a5c0a9

Browse files
zahzah
committed
feat(db-backend): add Elixir materialized DAP flow
Summary: - register Elixir as a materialized trace language and load recorder CTFS sidecar data through db-backend - add recorder sibling discovery, test-program symlink, FlowTestRunner coverage, just recipe, and cross-repo CI - pin the Elixir recorder and trace-format sibling SHAs used by the DAP flow validation Validation: - TMPDIR=/home/zahary/tmp/codex-work CODETRACER_ELIXIR_RECORDER_PATH=/home/zahary/metacraft/codetracer-elixir-recorder direnv exec . just test-elixir-flow - TMPDIR=/home/zahary/tmp/codex-work CODETRACER_ELIXIR_RECORDER_PATH=/home/zahary/metacraft/codetracer-elixir-recorder direnv exec . bash -lc 'cd src/db-backend && cargo test --test elixir_flow_dap_test e2e_codetracer_elixir_flow_dap -- --nocapture' - TMPDIR=/home/zahary/tmp/codex-work CODETRACER_ELIXIR_RECORDER_PATH=/home/zahary/metacraft/codetracer-elixir-recorder direnv exec . ./ci/test/elixir-flow-cross-repo.sh verify_elixir_flow_zero_test_guard - bash -n ci/test/elixir-flow-cross-repo.sh && bash -n scripts/detect-siblings.sh - direnv exec . python3 -c 'import yaml; yaml.safe_load(open(".github/workflows/elixir-flow-dap.yml"))' - TMPDIR=/home/zahary/tmp/codex-work direnv exec . bash -lc 'cd src/db-backend && cargo fmt -- --check'
1 parent a656cda commit 6a5c0a9

18 files changed

Lines changed: 1325 additions & 179 deletions

File tree

‎.github/sibling-pins‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ codetracer-evm-recorder 9dfae2a35fb4d2c18e9729f4b2f4242d0a533bf9 main
88
codetracer-flow-recorder 500943290fa9314d6dadbe68d224868bffd45294 main
99
codetracer-fuel-recorder 70ec83269b40279eb472de52d74da1316986fc3a main
1010
codetracer-js-recorder a54261badd04db86ff3266605edd35294ec5ca55 main
11+
codetracer-elixir-recorder a8630a06813919a50b7cbe3a0c50c24987e2b3fd main
1112
codetracer-leo-recorder c4f0300af6a2f560395f7f14f95646bde55b5aed main
1213
codetracer-miden-recorder de77543a76ba3311d28ad0cc9a74820548e93d14 main
1314
codetracer-move-recorder 046d6f681c43e3ab545e6a12e16b6e3b92a0a6e2 main
@@ -23,7 +24,7 @@ codetracer-shell-recorders e0d3247c7af50dfba91cce49f1df9dc59ad98870 HEAD
2324
codetracer-solana-recorder ae6fabf8b9a55f352cdbc62d3c73b6cda497a6ce main
2425
codetracer-specs 61aa7d740374fb241e49d896d3fc83bfb3d7a4e4 main
2526
codetracer-ton-recorder 4dac5e38e093bbf52f7b44a2fe0de8c93d5391de main
26-
codetracer-trace-format 4b2433f34a50fa94e0eb330df27d5fc6d78dfb26 main
27+
codetracer-trace-format f08b474075b80ebeb568faaad9aad2d4203a268c main
2728
codetracer-vscode-extension 90def1e0fc0dd6f4a3543817905b107b4e1dbbb0 main
2829
codetracer-wasmi-recorder 940344784979d8c3a9b59fa2ff1fcfd73cb5fbd3 HEAD
2930
codetracer-wasm-recorder 053dc64cf86a082d050d9f72f54e84285e8d2e82 wasm-tracing

‎.github/workflows/elixir-flow-dap.yml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
name: Elixir Materialized Trace DAP Flow
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- 'src/db-backend/**'
7+
- 'libs/ct-dap-client/**'
8+
- 'ci/test/elixir-flow-cross-repo.sh'
9+
- 'scripts/detect-siblings.sh'
10+
- 'justfile'
11+
- '.github/sibling-pins'
12+
- '.github/workflows/elixir-flow-dap.yml'
13+
push:
14+
branches: [main]
15+
paths:
16+
- 'src/db-backend/**'
17+
- 'libs/ct-dap-client/**'
18+
- 'ci/test/elixir-flow-cross-repo.sh'
19+
- 'scripts/detect-siblings.sh'
20+
- 'justfile'
21+
- '.github/sibling-pins'
22+
- '.github/workflows/elixir-flow-dap.yml'
23+
workflow_dispatch:
24+
inputs:
25+
elixir_recorder_ref:
26+
description: 'codetracer-elixir-recorder ref (default: .github/sibling-pins)'
27+
required: false
28+
default: ''
29+
trace_format_ref:
30+
description: 'codetracer-trace-format ref (default: .github/sibling-pins)'
31+
required: false
32+
default: ''
33+
34+
permissions:
35+
contents: read
36+
id-token: write
37+
38+
jobs:
39+
elixir-flow-dap:
40+
runs-on: [self-hosted, nixos]
41+
42+
steps:
43+
- name: Checkout codetracer
44+
uses: actions/checkout@v5
45+
with:
46+
submodules: recursive
47+
48+
- name: Generate CI token
49+
id: app-token
50+
uses: actions/create-github-app-token@v1
51+
with:
52+
app-id: ${{ vars.CI_TOKEN_PROVIDER_APP_ID }}
53+
private-key: ${{ secrets.CI_TOKEN_PROVIDER_PRIVATE_KEY }}
54+
owner: metacraft-labs
55+
56+
- name: Install Nix
57+
uses: ./.github/actions/setup-nix
58+
with:
59+
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
60+
cachix-cache: ${{ vars.CACHIX_CACHE }}
61+
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
62+
substituters: ${{ vars.SUBSTITUTERS }}
63+
gh-token: ${{ steps.app-token.outputs.token }}
64+
65+
- name: Resolve sibling refs
66+
id: refs
67+
run: |
68+
set -euo pipefail
69+
ELIXIR_REF="${{ github.event.inputs.elixir_recorder_ref }}"
70+
TRACE_FORMAT_REF="${{ github.event.inputs.trace_format_ref }}"
71+
if [[ -z "$ELIXIR_REF" ]]; then
72+
ELIXIR_REF="$(grep '^codetracer-elixir-recorder ' .github/sibling-pins | cut -d' ' -f2)"
73+
fi
74+
if [[ -z "$TRACE_FORMAT_REF" ]]; then
75+
TRACE_FORMAT_REF="$(grep '^codetracer-trace-format ' .github/sibling-pins | cut -d' ' -f2)"
76+
fi
77+
[[ "$ELIXIR_REF" =~ ^[0-9a-f]{40}$ ]] || { echo "codetracer-elixir-recorder ref must be a 40-char SHA: $ELIXIR_REF" >&2; exit 1; }
78+
[[ "$TRACE_FORMAT_REF" =~ ^[0-9a-f]{40}$ ]] || { echo "codetracer-trace-format ref must be a 40-char SHA: $TRACE_FORMAT_REF" >&2; exit 1; }
79+
echo "elixir-ref=$ELIXIR_REF" >> "$GITHUB_OUTPUT"
80+
echo "trace-format-ref=$TRACE_FORMAT_REF" >> "$GITHUB_OUTPUT"
81+
echo "codetracer-elixir-recorder ref: $ELIXIR_REF"
82+
echo "codetracer-trace-format ref: $TRACE_FORMAT_REF"
83+
84+
- name: Clone codetracer-elixir-recorder
85+
uses: metacraft-labs/metacraft-github-actions/clone-repo@main
86+
with:
87+
repo: metacraft-labs/codetracer-elixir-recorder
88+
ref: ${{ steps.refs.outputs.elixir-ref }}
89+
path: ${{ github.workspace }}/../codetracer-elixir-recorder
90+
gh-token: ${{ steps.app-token.outputs.token }}
91+
92+
- name: Replace trace-format with pinned sibling clone
93+
run: rm -rf libs/codetracer-trace-format
94+
95+
- name: Clone codetracer-trace-format
96+
uses: metacraft-labs/metacraft-github-actions/clone-repo@main
97+
with:
98+
repo: metacraft-labs/codetracer-trace-format
99+
ref: ${{ steps.refs.outputs.trace-format-ref }}
100+
path: ${{ github.workspace }}/libs/codetracer-trace-format
101+
gh-token: ${{ steps.app-token.outputs.token }}
102+
submodules: 'true'
103+
104+
- name: Run zero-test guard verification
105+
run: |
106+
set -euo pipefail
107+
export TMPDIR="${{ github.workspace }}/../tmp"
108+
mkdir -p "$TMPDIR"
109+
nix develop '.?submodules=1' --command \
110+
bash -lc './ci/test/elixir-flow-cross-repo.sh verify_elixir_flow_zero_test_guard'
111+
112+
- name: Run Elixir DAP flow integration
113+
run: |
114+
set -euo pipefail
115+
export TMPDIR="${{ github.workspace }}/../tmp"
116+
mkdir -p "$TMPDIR"
117+
export CODETRACER_ELIXIR_RECORDER_PATH="${{ github.workspace }}/../codetracer-elixir-recorder"
118+
nix develop '.?submodules=1' --command \
119+
bash -lc 'just test-elixir-flow'
120+
121+
- name: Upload full Elixir flow logs
122+
if: always()
123+
uses: actions/upload-artifact@v4
124+
with:
125+
name: elixir-flow-dap-logs
126+
path: target/elixir-flow-ci-logs/
127+
retention-days: 14
128+
if-no-files-found: ignore

‎ci/test/elixir-flow-cross-repo.sh‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
#!/usr/bin/env bash
2+
# Cross-repo Elixir materialized trace DAP flow runner.
3+
#
4+
# Subcommands:
5+
# e2e_cross_repo_ci_elixir_flow Build/use sibling recorder and run the real DAP flow test
6+
# verify_elixir_flow_zero_test_guard Prove zero discovered tests fail the CI guard
7+
set -euo pipefail
8+
9+
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
10+
LOG_DIR="$REPO_ROOT/target/elixir-flow-ci-logs"
11+
mkdir -p "$LOG_DIR"
12+
13+
usage() {
14+
cat <<'USAGE'
15+
usage: ci/test/elixir-flow-cross-repo.sh [e2e_cross_repo_ci_elixir_flow|verify_elixir_flow_zero_test_guard]
16+
USAGE
17+
}
18+
19+
fail() {
20+
echo "ERROR: $*" >&2
21+
exit 1
22+
}
23+
24+
guard_nonzero_tests() {
25+
local output="$1"
26+
if grep -Eq 'running 0 tests|0 passed; 0 failed|0 tests, 0 benchmarks' <<<"$output"; then
27+
echo "$output"
28+
echo "ERROR: Elixir flow test command discovered zero tests" >&2
29+
return 1
30+
fi
31+
if ! grep -q 'e2e_codetracer_elixir_flow_dap' <<<"$output"; then
32+
echo "$output"
33+
echo "ERROR: Elixir flow test output did not include e2e_codetracer_elixir_flow_dap" >&2
34+
return 1
35+
fi
36+
}
37+
38+
source_sibling_detection() {
39+
# shellcheck disable=SC1091
40+
source "$REPO_ROOT/scripts/detect-siblings.sh" "$REPO_ROOT" >/dev/null
41+
}
42+
43+
resolve_elixir_recorder() {
44+
source_sibling_detection
45+
: "${CODETRACER_ELIXIR_RECORDER_PATH:=${REPO_ROOT}/../codetracer-elixir-recorder}"
46+
[[ -d $CODETRACER_ELIXIR_RECORDER_PATH ]] ||
47+
fail "missing codetracer-elixir-recorder checkout: $CODETRACER_ELIXIR_RECORDER_PATH"
48+
49+
local debug_bin="$CODETRACER_ELIXIR_RECORDER_PATH/target/debug/codetracer-elixir-recorder"
50+
local release_bin="$CODETRACER_ELIXIR_RECORDER_PATH/target/release/codetracer-elixir-recorder"
51+
52+
echo "Building codetracer-elixir-recorder in $CODETRACER_ELIXIR_RECORDER_PATH"
53+
if command -v direnv >/dev/null 2>&1 && [[ -f "$CODETRACER_ELIXIR_RECORDER_PATH/.envrc" ]]; then
54+
direnv exec "$CODETRACER_ELIXIR_RECORDER_PATH" \
55+
cargo build --locked --manifest-path "$CODETRACER_ELIXIR_RECORDER_PATH/Cargo.toml"
56+
else
57+
cargo build --locked --manifest-path "$CODETRACER_ELIXIR_RECORDER_PATH/Cargo.toml"
58+
fi
59+
if [[ -x $debug_bin ]]; then
60+
export CODETRACER_ELIXIR_RECORDER_BIN="$debug_bin"
61+
elif [[ -x $release_bin ]]; then
62+
export CODETRACER_ELIXIR_RECORDER_BIN="$release_bin"
63+
else
64+
fail "recorder build succeeded but no recorder binary was found"
65+
fi
66+
}
67+
68+
print_pin_summary() {
69+
local pins="$REPO_ROOT/.github/sibling-pins"
70+
if [[ -f $pins ]]; then
71+
grep -E '^(codetracer-elixir-recorder|codetracer-trace-format) ' "$pins" || true
72+
fi
73+
if [[ -d "$CODETRACER_ELIXIR_RECORDER_PATH/.git" ]]; then
74+
echo "codetracer-elixir-recorder HEAD: $(git -C "$CODETRACER_ELIXIR_RECORDER_PATH" rev-parse HEAD)"
75+
fi
76+
if [[ -d "$REPO_ROOT/libs/codetracer-trace-format/.git" ]]; then
77+
echo "codetracer-trace-format HEAD: $(git -C "$REPO_ROOT/libs/codetracer-trace-format" rev-parse HEAD)"
78+
fi
79+
}
80+
81+
run_elixir_flow() {
82+
resolve_elixir_recorder
83+
print_pin_summary
84+
export TMPDIR="${TMPDIR:-/home/zahary/tmp/codex-work}"
85+
mkdir -p "$TMPDIR"
86+
87+
local log_file="$LOG_DIR/e2e_cross_repo_ci_elixir_flow.log"
88+
local output status
89+
set +e
90+
output="$(
91+
cd "$REPO_ROOT/src/db-backend" &&
92+
cargo test --test elixir_flow_dap_test e2e_codetracer_elixir_flow_dap -- --nocapture 2>&1
93+
)"
94+
status=$?
95+
set -e
96+
printf '%s\n' "$output" | tee "$log_file"
97+
[[ $status -eq 0 ]] || exit "$status"
98+
guard_nonzero_tests "$output" || exit 1
99+
}
100+
101+
verify_zero_test_guard() {
102+
local output status
103+
set +e
104+
output="$(
105+
cd "$REPO_ROOT/src/db-backend" &&
106+
cargo test --test elixir_flow_dap_test __codetracer_elixir_no_such_test__ -- --nocapture 2>&1
107+
)"
108+
status=$?
109+
set -e
110+
if [[ $status -ne 0 ]]; then
111+
printf '%s\n' "$output"
112+
fail "zero-test guard setup failed before guard could inspect cargo output"
113+
fi
114+
if guard_nonzero_tests "$output" >/dev/null 2>&1; then
115+
printf '%s\n' "$output"
116+
fail "zero-test guard accepted a vacuous cargo test run"
117+
fi
118+
printf '%s\n' "$output" >"$LOG_DIR/verify_elixir_flow_zero_test_guard.log"
119+
echo "verify_elixir_flow_zero_test_guard: guard rejected zero discovered tests"
120+
}
121+
122+
case "${1:-e2e_cross_repo_ci_elixir_flow}" in
123+
e2e_cross_repo_ci_elixir_flow)
124+
run_elixir_flow
125+
;;
126+
verify_elixir_flow_zero_test_guard)
127+
verify_zero_test_guard
128+
;;
129+
-h | --help)
130+
usage
131+
;;
132+
*)
133+
usage >&2
134+
exit 2
135+
;;
136+
esac

‎justfile‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -652,6 +652,15 @@ test-ruby-flow:
652652
cd src/db-backend && cargo nextest run --no-capture test_ruby_flow
653653
echo "Ruby flow test passed!"
654654
655+
# Elixir materialized trace DAP flow integration test (DB-based, no rr required)
656+
# Uses CODETRACER_ELIXIR_RECORDER_PATH for explicit sibling discovery.
657+
test-elixir-flow:
658+
#!/usr/bin/env bash
659+
set -euo pipefail
660+
echo "Running Elixir materialized trace DAP flow integration test..."
661+
./ci/test/elixir-flow-cross-repo.sh e2e_cross_repo_ci_elixir_flow
662+
echo "Elixir flow test passed!"
663+
655664
# Noir flow/omniscience integration test (DB-based, no rr required)
656665
test-noir-flow:
657666
#!/usr/bin/env bash

‎libs/ct-dap-client/src/test_support/flow_runner.rs‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,16 @@ impl FlowTestRunner {
242242
/// are self-contained: the trace folder has trace.json/trace.bin +
243243
/// trace_metadata.json and db-backend auto-detects the format.
244244
pub fn new_db_trace(db_backend_bin: &Path, trace_dir: &Path) -> Result<Self, BoxError> {
245+
Self::new_db_trace_with_timeout(db_backend_bin, trace_dir, Duration::from_secs(10))
246+
}
247+
248+
/// Spawn db-backend for a DB trace and wait up to `startup_timeout` for
249+
/// the initial stopped event.
250+
pub fn new_db_trace_with_timeout(
251+
db_backend_bin: &Path,
252+
trace_dir: &Path,
253+
startup_timeout: Duration,
254+
) -> Result<Self, BoxError> {
245255
let mut client = DapStdioClient::spawn(db_backend_bin)?;
246256

247257
let _caps = client.initialize()?;
@@ -252,7 +262,7 @@ impl FlowTestRunner {
252262
})?;
253263

254264
client.configuration_done()?;
255-
client.wait_for_stopped(Duration::from_secs(10))?;
265+
client.wait_for_stopped(startup_timeout)?;
256266

257267
Ok(FlowTestRunner {
258268
client,
@@ -316,8 +326,13 @@ impl FlowTestRunner {
316326

317327
let move_state = self.client.dap_continue()?;
318328
println!(
319-
" Stopped at {}:{}",
320-
move_state.location.path, move_state.location.line
329+
" Stopped at {}:{} key={} ticks={} fn={}..{}",
330+
move_state.location.path,
331+
move_state.location.line,
332+
move_state.location.key,
333+
move_state.location.rr_ticks.0,
334+
move_state.location.function_first,
335+
move_state.location.function_last
321336
);
322337

323338
// Load flow and verify variables at this breakpoint.

‎libs/ct-dap-client/src/types/common.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ pub enum Lang {
2626
Small,
2727
PythonDb,
2828
Unknown,
29+
Elixir,
2930
}
3031

3132
/// Kinds of I/O or log events (repr(u8), matching codetracer_trace_types).

‎scripts/detect-siblings.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ _ct_try_workspace_root() {
5555
[ -d "$candidate/codetracer-python-recorder" ] ||
5656
[ -d "$candidate/codetracer-ruby-recorder" ] ||
5757
[ -d "$candidate/codetracer-js-recorder" ] ||
58+
[ -d "$candidate/codetracer-elixir-recorder" ] ||
5859
[ -d "$candidate/codetracer-shell-recorders" ] ||
5960
[ -d "$candidate/codetracer-wasm-recorder" ] ||
6061
[ -d "$candidate/codetracer-native-test-programs" ] ||
@@ -156,6 +157,31 @@ if [ -n "$_CT_WORKSPACE_ROOT" ] && [ -d "$_CT_WORKSPACE_ROOT/codetracer-js-recor
156157
_ct_detect_summary "codetracer-js-recorder"
157158
fi
158159

160+
# --- codetracer-elixir-recorder ---
161+
_ct_elixir_recorder_path="${CODETRACER_ELIXIR_RECORDER_PATH:-}"
162+
if [ -z "$_ct_elixir_recorder_path" ] &&
163+
[ -n "$_CT_WORKSPACE_ROOT" ] &&
164+
[ -d "$_CT_WORKSPACE_ROOT/codetracer-elixir-recorder" ]; then
165+
_ct_elixir_recorder_path="$_CT_WORKSPACE_ROOT/codetracer-elixir-recorder"
166+
export CODETRACER_ELIXIR_RECORDER_PATH="$_ct_elixir_recorder_path"
167+
fi
168+
if [ -n "$_ct_elixir_recorder_path" ] && [ -d "$_ct_elixir_recorder_path" ]; then
169+
for _ct_elixir_profile in debug release; do
170+
_ct_elixir_bin="$_ct_elixir_recorder_path/target/$_ct_elixir_profile/codetracer-elixir-recorder"
171+
if [ -x "$_ct_elixir_bin" ]; then
172+
export CODETRACER_ELIXIR_RECORDER_BIN="$_ct_elixir_bin"
173+
export PATH="$_ct_elixir_recorder_path/target/$_ct_elixir_profile:$PATH"
174+
_ct_detect_summary "codetracer-elixir-recorder ($_ct_elixir_profile build)"
175+
break
176+
fi
177+
done
178+
if [ -z "${CODETRACER_ELIXIR_RECORDER_BIN:-}" ]; then
179+
_ct_detect_summary "codetracer-elixir-recorder (repo present, binary not built)"
180+
fi
181+
unset _ct_elixir_profile _ct_elixir_bin
182+
fi
183+
unset _ct_elixir_recorder_path
184+
159185
# --- codetracer-shell-recorders ---
160186
if [ -n "$_CT_WORKSPACE_ROOT" ] && [ -d "$_CT_WORKSPACE_ROOT/codetracer-shell-recorders/bash-recorder" ]; then
161187
export PATH="$_CT_WORKSPACE_ROOT/codetracer-shell-recorders/bash-recorder:$_CT_WORKSPACE_ROOT/codetracer-shell-recorders/zsh-recorder:$PATH"

0 commit comments

Comments
 (0)