Add testbot: AI-powered test generation and review response

.github/workflows/testbot-respond.yaml

@@ -0,0 +1,92 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Responds to /testbot commands in review comments and PR comments on
+# AI-generated PRs. Delegates fixes to Claude Code CLI, posts inline
+# replies, and resolves addressed threads.
+name: Testbot - Respond to Reviews
+
+on:
+  pull_request_review_comment:
+    types: [created]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: testbot-respond-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  respond:
+    # Label gate only — /testbot trigger, author, and fork checks are in respond.py.
+    if: contains(github.event.pull_request.labels.*.name, 'ai-generated')
+    timeout-minutes: 15
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout PR branch
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          fetch-depth: 0
+          token: ${{ secrets.SVC_OSMO_CI_TOKEN }}
+
+      - name: Setup Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.10'
+
+      - name: Setup Node.js
+        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
+        with:
+          node-version: 20
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          package_json_file: src/ui/package.json
+
+      - name: Install UI dependencies
+        working-directory: src/ui
+        run: pnpm install --frozen-lockfile
+
+      - name: Setup Bazel
+        uses: bazel-contrib/setup-bazel@4fd964a13a440a8aeb0be47350db2fc640f19ca8
+        with:
+          bazelisk-cache: true
+          bazelisk-version: 1.27.0
+          disk-cache: ${{ github.workflow }}
+          repository-cache: true
+
+      - name: Configure git
+        run: |
+          git config user.name "testbot[bot]"
+          git config user.email "testbot[bot]@users.noreply.github.com"
+
+      - name: Respond to review comments
+        run: |
+          PYTHONPATH=src/scripts python -m testbot.respond \
+            --pr-number ${{ github.event.pull_request.number }} \
+            --trigger-phrase /testbot
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.NVIDIA_NIM_KEY }}
+          ANTHROPIC_BASE_URL: https://inference-api.nvidia.com
+          ANTHROPIC_MODEL: aws/anthropic/claude-opus-4-5
+          DISABLE_PROMPT_CACHING: "1"
+          TESTBOT_MAX_TURNS: "50"
+          TESTBOT_MAX_RESPONSES: "10"
+          GH_TOKEN: ${{ secrets.SVC_OSMO_CI_TOKEN }}

.github/workflows/testbot.yaml

@@ -0,0 +1,141 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Testbot: analyzes coverage gaps, generates tests via Claude Code,
+# validates them, and opens a PR for human review. Runs daily on weekdays.
+name: Testbot
+
+on:
+  schedule:
+    - cron: '0 6 * * 1-5'  # Weekdays 6 AM UTC
+  workflow_dispatch:
+    inputs:
+      max_targets:
+        description: 'Max files to target'
+        default: '1'
+      max_uncovered:
+        description: 'Max uncovered lines per target (0 = no cap)'
+        default: '300'
+      max_turns:
+        description: 'Max Claude Code agent turns'
+        default: '50'
+      timeout_minutes:
+        description: 'Workflow timeout in minutes'
+        default: '30'
+      model:
+        description: 'LLM model name on NVIDIA gateway'
+        default: 'aws/anthropic/claude-opus-4-5'
+      dry_run:
+        description: 'Generate but do not create PR'
+        type: boolean
+        default: false
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: testbot
+  cancel-in-progress: true
+
+jobs:
+  generate-tests:
+    timeout-minutes: ${{ fromJSON(inputs.timeout_minutes || '30') }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          lfs: true
+          fetch-depth: 0
+          token: ${{ secrets.SVC_OSMO_CI_TOKEN }}
+
+      - name: Setup Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.10'
+
+      - name: Setup Node.js
+        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
+        with:
+          node-version: 20
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          package_json_file: src/ui/package.json
+
+      - name: Install UI dependencies
+        working-directory: src/ui
+        run: pnpm install --frozen-lockfile
+
+      - name: Setup Bazel
+        uses: bazel-contrib/setup-bazel@4fd964a13a440a8aeb0be47350db2fc640f19ca8
+        with:
+          bazelisk-cache: true
+          bazelisk-version: 1.27.0
+          disk-cache: ${{ github.workflow }}
+          repository-cache: true
+          external-cache: |
+            manifest:
+              osmo_python_deps: src/locked_requirements.txt
+              osmo_tests_python_deps: src/tests/locked_requirements.txt
+              osmo_mypy_deps: bzl/mypy/locked_requirements.txt
+              pylint_python_deps: bzl/linting/locked_requirements.txt
+              io_bazel_rules_go: src/runtime/go.mod
+              bazel_gazelle: src/runtime/go.sum
+
+      - name: Configure git
+        run: |
+          git config user.name "testbot[bot]"
+          git config user.email "testbot[bot]@users.noreply.github.com"
+
+      - name: Select coverage targets
+        run: |
+          python src/scripts/testbot/coverage_targets.py \
+            --max-targets ${{ inputs.max_targets || '1' }} \
+            --max-uncovered ${{ inputs.max_uncovered || '300' }} \
+            > "$RUNNER_TEMP/targets.txt"
+          echo "::group::Selected targets"
+          cat "$RUNNER_TEMP/targets.txt"
+          echo "::endgroup::"
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Generate tests
+        if: inputs.dry_run != true
+        run: |
+          TARGETS=$(cat "$RUNNER_TEMP/targets.txt")
+          npx @anthropic-ai/claude-code@latest --print \
+            --model "$ANTHROPIC_MODEL" \
+            --allowedTools "Read,Edit,Write,Bash(bazel:*),Bash(pnpm:*),Glob,Grep" \
+            --max-turns ${{ inputs.max_turns || '50' }} \
+            "$(cat src/scripts/testbot/TESTBOT_PROMPT.md)
+
+          Coverage targets:
+          $TARGETS"
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.NVIDIA_NIM_KEY }}
+          ANTHROPIC_BASE_URL: https://inference-api.nvidia.com
+          ANTHROPIC_MODEL: ${{ inputs.model || 'aws/anthropic/claude-opus-4-5' }}
+          DISABLE_PROMPT_CACHING: "1"
+
+      - name: Create PR
+        if: inputs.dry_run != true
+        run: |
+          PYTHONPATH=src/scripts python -m testbot.create_pr
+        env:
+          GH_TOKEN: ${{ secrets.SVC_OSMO_CI_TOKEN }}

AGENTS.md

@@ -193,6 +193,11 @@ Features: Tab completion (shtab), response formatting (`formatters.py`), spec ed
 - `backend_worker.py` — Job execution engine for backend tasks
 - `backend_test_runner/` — Test orchestration for backend validation
 - `utils/node_validation_test/` — GPU validation (nvidia-smi, tflops benchmark, stuck pod detection)
+
+### Testbot (`scripts/testbot/`)
+
+AI-powered test generation and review response using Claude Code CLI. See `scripts/testbot/README.md` for details.
+
 ### Tests
 
 
@@ -204,6 +209,7 @@ Features: Tab completion (shtab), response formatting (`formatters.py`), spec ed
 | `utils/*_test.go`                           | Go testing                     | Go utility tests (roles, postgres, redis)                                                                                                                  |
 | `ui/src/**/*.test.ts`                       | Vitest                         | Frontend unit tests                                                                                                                                        |
 | `ui/e2e/`                                   | Playwright                     | Frontend E2E tests with page object models                                                                                                                 |
+| `scripts/testbot/tests/`                    | unittest                       | Testbot unit tests (coverage targets, guardrails, respond, create_pr)                                                                                      |
 
 ### Key Architecture Patterns
 

src/scripts/testbot/BUILD

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+load("//bzl:py.bzl", "osmo_py_library")
+
+osmo_py_library(
+    name = "testbot",
+    srcs = glob(["*.py"]),
+    data = ["TESTBOT_PROMPT.md"],
+    visibility = ["//visibility:public"],
+)