Auto-generate protobuf and Connect RPC files via hatch build hook (#3631)

Remove checked-in generated protobuf and Connect RPC files, replacing
them with automatic generation via a hatch build hook. This eliminates
the need to manually regenerate and commit these files when .proto
sources change.

The custom build hook (hatch_build.py) runs during package installation
and wheel builds, regenerating all *_pb2.py, *_pb2.pyi, and *_connect.py
files from their .proto sources if the sources are newer than the
outputs. This ensures generated files are always in sync with their
sources without requiring manual steps or git commits.

Updated .gitignore to exclude the generated files, and removed the
manual dashboard build fixture from conftest.py since the build hook now
handles all code generation automatically.

Author: rjpower

.github/workflows/iris-unit-tests.yaml

@@ -31,6 +31,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
       - name: Install uv
         uses: astral-sh/setup-uv@v7
         with:
@@ -58,6 +63,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
       - name: Install uv
         uses: astral-sh/setup-uv@v7
         with:

.github/workflows/levanter-tests.yaml

@@ -30,6 +30,10 @@ jobs:
           python-version: ${{ matrix.python-version }}
           enable-cache: true
           working-directory: lib/levanter
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
       - name: Set up Python
         run: uv python install
       - name: Install dependencies
@@ -57,6 +61,10 @@ jobs:
           python-version: ${{ matrix.python-version }}
           enable-cache: true
           working-directory: lib/levanter
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
       - name: Set up Python
         run: uv python install
       - name: Install dependencies
@@ -83,6 +91,10 @@ jobs:
           python-version: ${{ matrix.python-version }}
           enable-cache: true
           working-directory: lib/levanter
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
       - name: Set up Python
         run: uv python install
       - name: Install dependencies
@@ -110,6 +122,10 @@ jobs:
           python-version: ${{ matrix.python-version }}
           enable-cache: true
           working-directory: lib/levanter
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
       - name: Set up Python
         run: uv python install
       - name: Install dependencies
@@ -201,4 +217,22 @@ jobs:
             -v /tmp/uv-cache:/tmp/uv-cache:rw \
             -w /workspace \
             $DOCKER_IMAGE \
-            bash -c "cp -a /workspace-src/. /workspace/ && cd /workspace && timeout --kill-after=5 --signal=TERM 890 uv run --package levanter --frozen --group test --with 'jax[tpu]==$JAX_VERSION' pytest lib/levanter/tests -m 'not entry and not ray and not slow and not torch' --ignore=lib/levanter/tests/test_audio.py --ignore=lib/levanter/tests/test_new_cache.py --ignore=lib/levanter/tests/test_hf_checkpoints.py --ignore=lib/levanter/tests/test_hf_gpt2_serialize.py --ignore=lib/levanter/tests/test_gdn_layer.py -v --tb=short --log-cli-level=WARNING --durations=20"
+            bash -c "\
+              # Install Node.js in userspace if not present (needed for protobuf generation during uv sync)
+              if ! command -v npx >/dev/null 2>&1; then \
+                echo '::group::Installing Node.js in userspace'; \
+                curl -fsSL https://nodejs.org/dist/v22.16.0/node-v22.16.0-linux-x64.tar.xz | tar -xJ -C /tmp && \
+                export PATH=/tmp/node-v22.16.0-linux-x64/bin:\$PATH; \
+                echo '::endgroup::'; \
+              fi && \
+              cp -a /workspace-src/. /workspace/ && cd /workspace && \
+              timeout --kill-after=5 --signal=TERM 890 \
+              uv run --package levanter --frozen --group test --with 'jax[tpu]==$JAX_VERSION' \
+              pytest lib/levanter/tests \
+              -m 'not entry and not ray and not slow and not torch' \
+              --ignore=lib/levanter/tests/test_audio.py \
+              --ignore=lib/levanter/tests/test_new_cache.py \
+              --ignore=lib/levanter/tests/test_hf_checkpoints.py \
+              --ignore=lib/levanter/tests/test_hf_gpt2_serialize.py \
+              --ignore=lib/levanter/tests/test_gdn_layer.py \
+              -v --tb=short --log-cli-level=WARNING --durations=20"

.github/workflows/marin-docs.yaml

@@ -24,6 +24,11 @@ jobs:
           python-version: "3.11"
           enable-cache: true
 
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
       - name: Set up Python
         run: uv python install
 

.github/workflows/marin-itest.yaml

@@ -31,6 +31,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
       - name: Install uv
         uses: astral-sh/setup-uv@v7
         with:

.github/workflows/marin-lint-and-format.yaml

@@ -24,5 +24,10 @@ jobs:
         python-version: "3.11"
         enable-cache: true
 
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '22'
+
     - name: Run pre-commit checks
       run: ./infra/pre-commit.py --all-files

.github/workflows/marin-unit-tests.yaml

@@ -39,7 +39,7 @@ jobs:
           uv sync --package marin --extra cpu --extra dedup --group test --frozen
 
       - name: Set up Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
 

.github/workflows/zephyr-unit-tests.yaml

@@ -31,6 +31,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:

.gitignore

@@ -234,3 +234,8 @@ scr/*
 .agents/tmp/
 .codex
 .entire
+
+# Generated protobuf and Connect RPC files (rebuilt by hatch build hook)
+lib/iris/src/iris/rpc/*_pb2.py
+lib/iris/src/iris/rpc/*_pb2.pyi
+lib/iris/src/iris/rpc/*_connect.py

.readthedocs.yaml

@@ -4,6 +4,7 @@ build:
   os: "ubuntu-24.04"
   tools:
     python: "3.11"
+    nodejs: "22"
   commands:
     - pip install uv
     - uv sync --group docs --package marin

Makefile

@@ -205,5 +205,20 @@ setup_pre_commit:
 	chmod +x $$HOOK_PATH; \
 	echo "Installed git pre-commit hook -> $$HOOK_PATH"
 
-dev_setup: install_uv install_gcloud get_secret_key get_ray_auth_token setup_pre_commit
+install_node:
+	@if command -v node > /dev/null 2>&1; then \
+		echo "Node.js $$(node --version) is already installed."; \
+	elif command -v brew > /dev/null 2>&1; then \
+		echo "Installing Node.js via Homebrew..."; \
+		brew install node; \
+	elif command -v apt-get > /dev/null 2>&1; then \
+		echo "Installing Node.js via apt..."; \
+		curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash - && \
+		sudo apt-get install -y nodejs; \
+	else \
+		echo "Cannot auto-install Node.js. Please install manually: https://nodejs.org/"; \
+		exit 1; \
+	fi
+
+dev_setup: install_uv install_gcloud install_node get_secret_key get_ray_auth_token setup_pre_commit
 	@echo "Dev setup complete."