Skip to content

Commit 402a94b

Browse files
committed
Gate v2 generated stubs
- Gate `build-v2` and `pre-flight-v2` on generated drift - Add uv pin preflight before v2 sync and stub generation - Document when to run `make py-stubs-v2` and what to commit
1 parent c16ad80 commit 402a94b

7 files changed

Lines changed: 127 additions & 31 deletions

File tree

.github/workflows/build-v2.yml

Lines changed: 5 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -250,29 +250,12 @@ jobs:
250250
--rootdir="$GITHUB_WORKSPACE/${{ env.PACKAGE_DIR }}" \
251251
"$GITHUB_WORKSPACE/${{ env.PACKAGE_DIR }}/tests/" -v
252252
253-
# Temporary: show whether v2 stub or docstring generation dirtied tracked files while we isolate CI drift
254-
- name: Report generated v2 file drift
255-
if: always()
253+
- name: Check generated v2 file drift
254+
# Keep this 3.13 condition in sync with the matrix so drift is checked once per run.
255+
if: matrix.python-version == '3.13'
256256
run: |
257-
targets=(
258-
python/generate_docstrings.py
259-
python/generate_stubs.py
260-
crates/pyo3
261-
':(glob)python/nautilus_trader/**/*.pyi'
262-
':(glob)crates/**/src/python/**/*.rs'
263-
)
264-
265-
changes="$(git status --short --untracked-files=all -- "${targets[@]}")"
266-
267-
if [ -z "$changes" ]; then
268-
echo "No generated v2 file drift detected"
269-
exit 0
270-
fi
271-
272-
echo "::warning::Tracked or untracked v2 build files changed during the job"
273-
printf '%s\n' "$changes"
274-
echo
275-
git diff --stat HEAD -- "${targets[@]}" || true
257+
make py-stubs-v2 V2_CARGO_TARGET_DIR=/home/runner/.cache/cargo-target/py-stubs-v2
258+
bash scripts/ci/check-v2-generated-drift.bash
276259
277260
- name: Upload wheel artifact
278261
uses: ./.github/actions/upload-artifact-wheel

CONTRIBUTING.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,5 +39,7 @@ To contribute, follow these steps:
3939

4040
- Follow the established coding practices in the [Developer Guide](https://nautilustrader.io/docs/developer_guide/index.html).
4141
- For documentation changes, follow the style guide in `docs/developer_guide/docs.md` (use sentence case for headings H2 and below).
42+
- For v2 PyO3 bindings or wrapped Rust docs, run `make py-stubs-v2` and commit the
43+
generated output. See [Generated Python artifacts](docs/developer_guide/rust.md#generated-python-artifacts).
4244
- Keep PRs small and focused for easier review.
4345
- Reference the relevant GitHub issue(s) in your PR comment.

Makefile

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,12 @@ LYCHEE_VERSION := $(shell bash scripts/cargo-tool-version.sh lychee)
2020
# Tool versions from tools.toml
2121
PREK_VERSION := $(shell bash scripts/tool-version.sh prek)
2222
UV_VERSION := $(shell bash scripts/uv-version.sh)
23+
UV_V2_REQUIRED_SPEC := $(shell awk -F'"' '\
24+
/^\[tool\.uv\]/ { in_section=1; next } \
25+
/^\[/ { in_section=0 } \
26+
in_section && /^[[:space:]]*required-version[[:space:]]*=/ { print $$2; exit } \
27+
' python/pyproject.toml)
28+
UV_V2_REQUIRED_VERSION := $(patsubst ==%,%,$(UV_V2_REQUIRED_SPEC))
2329

2430
V = 0 # 0 / 1 - verbose mode
2531
Q = $(if $(filter 1,$V),,@) # Quiet mode, suppress command output
@@ -32,6 +38,8 @@ VERBOSE ?= true
3238
# Set UV_SYNC_FLAGS= to make uv prune packages not in uv.lock
3339
UV_SYNC_FLAGS ?= --inexact
3440

41+
V2_CARGO_TARGET_DIR ?= $(CURDIR)/target-v2
42+
3543
PIP_AUDIT_IGNORE_FLAGS :=
3644

3745
# TARGET_DIR controls where cargo places build artifacts.
@@ -1037,19 +1045,40 @@ test-performance: #-- Run performance tests with codspeed benchmarking
10371045

10381046
.PHONY: sync-v2
10391047
sync-v2: #-- Sync v2 Python dependencies (without building the package)
1040-
$(info $(M) Syncing v2 Python dependencies...)
1048+
@if [ -z "$(UV_V2_REQUIRED_SPEC)" ]; then \
1049+
printf "$(RED)ERROR: Could not find required-version in python/pyproject.toml$(RESET)\n"; \
1050+
exit 1; \
1051+
fi
1052+
@if [ "$(UV_V2_REQUIRED_SPEC)" = "$(UV_V2_REQUIRED_VERSION)" ]; then \
1053+
printf "$(RED)ERROR: python/pyproject.toml required-version must use ==A.B.C, found $(UV_V2_REQUIRED_SPEC)$(RESET)\n"; \
1054+
exit 1; \
1055+
fi
1056+
@found="$$(uv --version 2>/dev/null | awk '{print $$2}' || true)"; \
1057+
if [ -z "$$found" ]; then \
1058+
printf "$(RED)ERROR: uv not found, ==$(UV_V2_REQUIRED_VERSION) required; run \`uv self update --version $(UV_V2_REQUIRED_VERSION)\` or prepend a matching binary to PATH.$(RESET)\n"; \
1059+
exit 1; \
1060+
fi; \
1061+
if [ "$$found" != "$(UV_V2_REQUIRED_VERSION)" ]; then \
1062+
printf "$(RED)ERROR: uv $$found found, ==$(UV_V2_REQUIRED_VERSION) required; run \`uv self update --version $(UV_V2_REQUIRED_VERSION)\` or prepend a matching binary to PATH.$(RESET)\n"; \
1063+
exit 1; \
1064+
fi
1065+
@printf "$(M) Syncing v2 Python dependencies...\n"
10411066
$Q cd python && VIRTUAL_ENV= uv sync --all-groups --all-extras --no-install-package nautilus-trader $(UV_SYNC_FLAGS)
10421067

10431068
.PHONY: build-debug-v2
10441069
build-debug-v2: sync-v2 #-- Build the v2 Python package in debug mode (also regenerates type stubs)
10451070
@$(MAKE) --no-print-directory py-stubs-v2
10461071
$(info $(M) Building v2 extension in debug mode...)
1047-
$Q cd python && VIRTUAL_ENV= CARGO_TARGET_DIR=../target-v2 uv run --no-sync maturin develop
1072+
$Q cd python && VIRTUAL_ENV= CARGO_TARGET_DIR=$(V2_CARGO_TARGET_DIR) uv run --no-sync maturin develop
10481073

10491074
.PHONY: py-stubs-v2
10501075
py-stubs-v2: sync-v2 #-- Regenerate v2 Python type stubs from Rust bindings
10511076
$(info $(M) Generating v2 Python type stubs...)
1052-
$Q cd python && VIRTUAL_ENV= CARGO_TARGET_DIR=$(CURDIR)/target-v2 uv run --no-sync python generate_stubs.py
1077+
$Q cd python && VIRTUAL_ENV= CARGO_TARGET_DIR=$(V2_CARGO_TARGET_DIR) uv run --no-sync python generate_stubs.py
1078+
1079+
.PHONY: check-v2-generated-drift
1080+
check-v2-generated-drift: #-- Check v2 generated stubs and docstrings are committed
1081+
$Q bash scripts/ci/check-v2-generated-drift.bash
10531082

10541083
.PHONY: update-v2
10551084
update-v2: cargo-update #-- Update v2 dependencies (cargo and uv)
@@ -1064,7 +1093,7 @@ pytest-v2: build-debug-v2 #-- Run v2 Python tests
10641093

10651094
.PHONY: pre-flight-v2
10661095
pre-flight-v2: export CARGO_TARGET_DIR=target-v2
1067-
pre-flight-v2: #-- Run comprehensive v2 pre-flight checks (format, check-code, cargo-test, build, pytest)
1096+
pre-flight-v2: #-- Run v2 pre-flight checks (format, tests, build, generated drift, audit)
10681097
$(info $(M) Running v2 pre-flight checks...)
10691098
@if ! git diff --quiet; then \
10701099
printf "$(RED)ERROR: You have unstaged changes$(RESET)\n"; \
@@ -1077,6 +1106,7 @@ pre-flight-v2: #-- Run comprehensive v2 pre-flight checks (format, check-code,
10771106
&& $(MAKE) --no-print-directory check-code EXTRA_FEATURES="capnp,hypersync" \
10781107
&& $(MAKE) --no-print-directory cargo-test-extras \
10791108
&& $(MAKE) --no-print-directory build-debug-v2 \
1109+
&& $(MAKE) --no-print-directory check-v2-generated-drift \
10801110
&& $(MAKE) --no-print-directory pytest-v2 \
10811111
&& $(MAKE) --no-print-directory security-audit \
10821112
$(call timer_end,Pre-flight)

README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -582,6 +582,10 @@ See the [Developer Guide](https://nautilustrader.io/docs/latest/developer_guide/
582582
>
583583
> Run `make build-debug` to compile after changes to Rust or Cython code for the most efficient development workflow.
584584

585+
After changes to v2 PyO3 bindings or wrapped Rust docs, run `make py-stubs-v2` and commit the
586+
generated `.pyi` files and wrapper docstrings. See
587+
[Generated Python artifacts](docs/developer_guide/rust.md#generated-python-artifacts).
588+
585589
### Testing with Rust
586590

587591
[cargo-nextest](https://nexte.st) is the standard Rust test runner for NautilusTrader.

docs/developer_guide/environment_setup.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,8 @@ Python dependencies are managed by [uv](https://docs.astral.sh/uv). The `[tool.u
220220
by `scripts/uv-version.sh` for Makefile, CI, and Docker builds. If your local uv drifts off the
221221
pin, `uv lock`/`uv sync` will fail with `Required uv version ... does not match the running
222222
version ...`. Run `make update-uv` to install the pinned version (or follow uv's own
223-
`uv self update <version>` hint).
223+
`uv self update <version>` hint). The v2 stub targets check the `python/pyproject.toml` pin
224+
before running `uv`; see [Generated Python artifacts](rust.md#generated-python-artifacts).
224225
- **`exclude-newer = "3 days"`**: `uv lock` ignores package versions published within the last
225226
3 days. This gives the community time to detect and quarantine compromised releases before they
226227
enter the lockfile. The value accepts an RFC 3339 timestamp (`"2026-03-30T00:00:00Z"`), a friendly

docs/developer_guide/rust.md

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -452,10 +452,48 @@ python = ["pyo3", "pyo3-stub-gen"]
452452
pyo3-stub-gen = { workspace = true, optional = true }
453453
```
454454

455-
**Regenerating stubs:** run `make py-stubs-v2` (or `python python/generate_stubs.py`)
456-
after changing annotations. The post-processor handles `py_` prefix stripping,
457-
`@property`/`@staticmethod`/`@classmethod` decoration, keyword escaping, deduplication,
458-
and ruff formatting.
455+
### Generated Python artifacts
456+
457+
The v2 Python surface commits two generated artifact types:
458+
459+
- Python type stubs under `python/nautilus_trader/**/*.pyi`.
460+
- PyO3 wrapper doc comments under `crates/**/src/python/**/*.rs`.
461+
462+
Run the generator with one command:
463+
464+
```bash
465+
make py-stubs-v2
466+
```
467+
468+
Run it after changing any Python-exposed Rust surface: `#[pyclass]`, `#[pymethods]`,
469+
`#[pyfunction]`, stub annotations, doc comments on wrapped core items, or adapter feature wiring.
470+
Commit every generated `.pyi` file and wrapper doc comment changed by the target with the source
471+
change. CI fails when committed output does not match regeneration. `make build-debug-v2` also
472+
regenerates these artifacts, but use `make py-stubs-v2` when you only need stubs and docstrings.
473+
474+
Wrapper `///` docs under `crates/**/src/python/**` are generated by
475+
`python/generate_docstrings.py` from the core Rust item docs. Do not hand-edit them. Edit the core
476+
docs and run `make py-stubs-v2`. The sync applies these transforms:
477+
478+
- `# Errors` and `# Safety` sections are copied as-is.
479+
- `# Panics` sections are dropped before they reach the Python API.
480+
- Intra-doc links are stripped.
481+
- Rust paths written with `::` become Python-style `.` paths.
482+
483+
The v2 target uses the uv version pinned by `required-version = "==0.11.26"` in
484+
`python/pyproject.toml`. If your local `uv` differs, `make sync-v2`, `make py-stubs-v2`, and
485+
`make build-debug-v2` fail before sync with the required version and update command. Run the
486+
`uv self update --version ...` command printed by the preflight, or prepend a matching `uv`
487+
binary to `PATH`.
488+
489+
Stub generation must compile the same optional Python surface that wheel builds expose.
490+
`python/generate_stubs.py` strips `extension-module` before running cargo, so features enabled only
491+
by `extension-module` in wheel builds must be appended explicitly in that script. Interactive
492+
Brokers uses this rule by appending `nautilus-interactive-brokers/gateway`, which keeps
493+
`DockerizedIBGateway` and `ContainerStatus` in the generated stubs.
494+
495+
The post-processor handles `py_` prefix stripping, `@property`/`@staticmethod`/`@classmethod`
496+
decoration, keyword escaping, deduplication, and ruff formatting.
459497

460498
### Constructor patterns
461499

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
targets=(
5+
python/generate_docstrings.py
6+
python/generate_stubs.py
7+
crates/pyo3
8+
':(glob)python/nautilus_trader/**/*.pyi'
9+
':(glob)crates/**/src/python/**/*.rs'
10+
)
11+
12+
git update-index -q --refresh
13+
14+
untracked="$(git ls-files --others --exclude-standard -- "${targets[@]}")"
15+
16+
if git diff --quiet HEAD -- "${targets[@]}" && [ -z "$untracked" ]; then
17+
echo "No generated v2 file drift detected"
18+
exit 0
19+
fi
20+
21+
echo "::error::Generated v2 files are out of sync"
22+
echo "Run \`make py-stubs-v2\` and commit the result."
23+
echo
24+
echo "Changed files:"
25+
git status --short --untracked-files=all -- "${targets[@]}"
26+
echo
27+
echo "Diff stat:"
28+
git diff --stat HEAD -- "${targets[@]}" || true
29+
30+
if [ -n "$untracked" ]; then
31+
echo
32+
echo "Untracked generated files:"
33+
printf '%s\n' "$untracked"
34+
fi
35+
36+
echo
37+
git diff --exit-code HEAD -- "${targets[@]}" || true
38+
exit 1

0 commit comments

Comments
 (0)