Repository navigation
test(#1778): add offline tests for pre-fix and pre-retro #4301
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Functional Tests | ||
|
Check warning on line 1 in .github/workflows/functional-tests.yml
|
||
| # PR-triggered functional tests use pull_request_target so fork PRs receive | ||
| # secrets. Authorization runs in a separate gate job (base checkout only) | ||
| # before the test job checks out the PR head — same pattern as fullsend e2e. | ||
| # | ||
| # Each change runs the full tier once, in the merge queue, on the commit that | ||
| # lands. PR runs use the release tier (one release:true case per agent) | ||
| # unless the PR carries the eval-full label. Applying eval-full does not | ||
| # start a run by itself (only ok-to-test does, so authorization stays | ||
| # tied to a fresh ok-to-test); it applies from the next push or | ||
| # ok-to-test run. There is no push-to-main run: | ||
| # the merge queue already tested that merge commit, and the nightly | ||
| # (functional-tests-nightly.yml) runs the full tier for every agent. | ||
| permissions: {} | ||
| on: | ||
| pull_request_target: | ||
| types: [opened, synchronize, reopened, labeled] | ||
| merge_group: | ||
| workflow_dispatch: | ||
| inputs: | ||
| tier: | ||
| description: 'Eval tier: auto (script default), full or release' | ||
| required: false | ||
| type: choice | ||
| options: [auto, full, release] | ||
| default: auto | ||
| workflow_call: | ||
| inputs: | ||
| fullsend_ref: | ||
| description: 'fullsend ref to build from (main or release tag v*)' | ||
| required: false | ||
| type: string | ||
| default: main | ||
| tier: | ||
| # Empty keeps eval/run-functional.sh's caller-derived default | ||
| # (release for a cross-repo call), so callers that omit it -- and | ||
| # callers pinned to an older copy of this workflow -- behave the same. | ||
| description: "Eval tier: '' or auto (script default), full or release" | ||
| required: false | ||
| type: string | ||
| default: '' | ||
| secrets: | ||
| E2E_GCP_WIF_PROVIDER: | ||
| required: false | ||
| E2E_GCP_SERVICE_ACCOUNT: | ||
| required: false | ||
| E2E_GCP_PROJECT_ID: | ||
| required: false | ||
| EVAL_GH_TOKEN: | ||
| required: false | ||
| outputs: | ||
| agents_sha: | ||
| description: 'Resolved commit SHA of the agents checkout (for cross-repo tagging)' | ||
| value: ${{ jobs.detect.outputs.agents_sha }} | ||
| concurrency: | ||
| group: >- | ||
| ${{ github.event_name == 'pull_request_target' | ||
| && format('functional-{0}', github.event.pull_request.number) | ||
| || github.repository != 'fullsend-ai/agents' | ||
| && format('functional-call-{0}', github.run_id) | ||
| || format('{0}-{1}', github.workflow, github.ref) }} | ||
| cancel-in-progress: >- | ||
| ${{ github.event_name == 'pull_request_target' | ||
| || github.ref != 'refs/heads/main' }} | ||
| jobs: | ||
| gate: | ||
| # Separate job so pull-requests: write stays out of the job that checks | ||
| # out fork head and runs tests with secrets. | ||
| # Never checkout github.event.pull_request.head.sha here. | ||
| if: >- | ||
| github.event_name == 'pull_request_target' && | ||
| (github.event.action != 'labeled' || github.event.label.name == 'ok-to-test') | ||
| runs-on: ubuntu-24.04 | ||
| timeout-minutes: 5 | ||
| permissions: | ||
| contents: read | ||
| pull-requests: write | ||
| outputs: | ||
| authorized: ${{ steps.auth.outputs.authorized }} | ||
| steps: | ||
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | ||
| with: | ||
| ref: ${{ github.sha }} # Base branch only — never checkout PR head in gate | ||
| - name: Check PR authorization | ||
| id: auth | ||
| uses: ./.github/actions/check-e2e-authorization | ||
| with: | ||
| pr_number: ${{ github.event.pull_request.number }} | ||
| repository: ${{ github.repository }} | ||
| pr_updated_at: ${{ github.event.pull_request.updated_at }} | ||
| event_action: ${{ github.event.action }} | ||
| pr_author_association: ${{ github.event.pull_request.author_association }} | ||
| pr_author_login: ${{ github.event.pull_request.user.login }} | ||
| label_actor_login: ${{ github.event.sender.login }} | ||
| # --------------------------------------------------------------------------- | ||
| # Detect which agents need functional testing based on changed files. | ||
| # Lightweight job: checkout + yq + select-eval-agents.sh → JSON array. | ||
| # --------------------------------------------------------------------------- | ||
| detect: | ||
| needs: gate | ||
| if: >- | ||
| !cancelled() && | ||
| github.repository_owner == 'fullsend-ai' && | ||
| (github.event_name != 'pull_request_target' || needs.gate.outputs.authorized == 'true') | ||
| runs-on: ubuntu-24.04 | ||
| timeout-minutes: 5 | ||
| permissions: | ||
| contents: read | ||
| outputs: | ||
| agents: ${{ steps.agents.outputs.agents }} | ||
| agents_sha: ${{ steps.checkout.outputs.commit }} | ||
| steps: | ||
| - name: Get changed files | ||
| id: changes | ||
| if: >- | ||
| github.repository == 'fullsend-ai/agents' && | ||
| (github.event_name == 'pull_request_target' || | ||
| github.event_name == 'merge_group') | ||
| env: | ||
| GH_TOKEN: ${{ github.token }} | ||
| EVENT_NAME: ${{ github.event_name }} | ||
| PR_NUMBER: ${{ github.event.pull_request.number }} | ||
| REPO: ${{ github.repository }} | ||
| MERGE_GROUP_BASE: ${{ github.event.merge_group.base_sha }} | ||
| MERGE_GROUP_HEAD: ${{ github.event.merge_group.head_sha }} | ||
| run: | | ||
| if [ "$EVENT_NAME" = "merge_group" ]; then | ||
| FILES=$(gh api "repos/${REPO}/compare/${MERGE_GROUP_BASE}...${MERGE_GROUP_HEAD}" --jq '.files[].filename') || { | ||
| echo "::warning::Failed to fetch merge group files — running all functional tests as a precaution" | ||
| exit 0 | ||
| } | ||
| FILE_COUNT=$(echo "$FILES" | wc -l) | ||
| if [ "$FILE_COUNT" -ge 300 ]; then | ||
| echo "::warning::Compare API returned $FILE_COUNT files (possible truncation at 300) — running all functional tests as a precaution" | ||
| exit 0 | ||
| fi | ||
| else | ||
| FILES=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename') || { | ||
| echo "::warning::Failed to fetch PR files — running all functional tests as a precaution" | ||
| exit 0 | ||
| } | ||
| FILE_COUNT=$(echo "$FILES" | wc -l) | ||
| if [ "$FILE_COUNT" -ge 300 ]; then | ||
| echo "::warning::PR file list returned $FILE_COUNT files (possible truncation) — running all functional tests as a precaution" | ||
| exit 0 | ||
| fi | ||
| fi | ||
| DELIM="ghadelim_$(openssl rand -hex 8)" | ||
| { | ||
| echo "files<<${DELIM}" | ||
| echo "$FILES" | ||
| echo "${DELIM}" | ||
| } >> "$GITHUB_OUTPUT" | ||
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | ||
| id: checkout | ||
| with: | ||
| repository: ${{ github.repository != 'fullsend-ai/agents' && 'fullsend-ai/agents' || github.repository }} | ||
| ref: >- | ||
| ${{ github.repository != 'fullsend-ai/agents' | ||
| && 'main' | ||
| || (github.event_name == 'pull_request_target' | ||
| && github.event.pull_request.head.sha | ||
| || github.sha) }} | ||
| persist-credentials: false | ||
| allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }} | ||
| - name: Install yq | ||
| run: | | ||
| curl -sSfL "https://github.com/mikefarah/yq/releases/download/v4.47.1/yq_linux_amd64" -o /usr/local/bin/yq | ||
| echo "0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb /usr/local/bin/yq" | sha256sum -c | ||
| chmod +x /usr/local/bin/yq | ||
| - name: Select eval agents | ||
| id: agents | ||
| env: | ||
| CHANGED_FILES: ${{ steps.changes.outputs.files }} | ||
| run: | | ||
| if [ -n "$CHANGED_FILES" ]; then | ||
| AGENTS=$(echo "$CHANGED_FILES" | bash .github/scripts/select-eval-agents.sh) | ||
| else | ||
| # No diff (dispatch, workflow_call, or a failed diff fetch): run all | ||
| # agents with eval configs | ||
| AGENTS=$(for d in eval/*/eval.yaml; do basename "$(dirname "$d")"; done) | ||
| fi | ||
| JSON=$(echo "$AGENTS" | jq -Rsc '[split("\n")[] | select(length > 0)]') | ||
| echo "agents=$JSON" >> "$GITHUB_OUTPUT" | ||
| echo "Selected agents: $JSON" | ||
| # --------------------------------------------------------------------------- | ||
| # Run functional tests — one matrix leg per agent, in parallel. | ||
| # Each leg does full setup (python, fullsend, podman, etc.) independently. | ||
| # --------------------------------------------------------------------------- | ||
| functional-tests: | ||
| needs: [gate, detect] | ||
| if: >- | ||
| !cancelled() && | ||
| github.repository_owner == 'fullsend-ai' && | ||
| needs.detect.outputs.agents != '[]' && | ||
| (github.event_name != 'pull_request_target' || needs.gate.outputs.authorized == 'true') | ||
| runs-on: ubuntu-24.04 | ||
| # The full-tier review leg takes 39-43 min (9 live cases); 60 leaves | ||
| # headroom. The release tier runs one case per agent. | ||
| timeout-minutes: 60 | ||
| permissions: | ||
| contents: read | ||
| id-token: write | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| agent: ${{ fromJSON(needs.detect.outputs.agents) }} | ||
| steps: | ||
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | ||
| with: | ||
| repository: ${{ github.repository != 'fullsend-ai/agents' && 'fullsend-ai/agents' || github.repository }} | ||
| ref: >- | ||
| ${{ github.repository != 'fullsend-ai/agents' | ||
| && (needs.detect.outputs.agents_sha || 'main') | ||
| || (github.event_name == 'pull_request_target' | ||
| && github.event.pull_request.head.sha | ||
| || github.sha) }} | ||
| persist-credentials: false | ||
| allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }} | ||
| submodules: true | ||
| - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | ||
| with: | ||
| python-version: "3.12" | ||
| - name: Install uv | ||
| uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 | ||
| - name: Install agent-eval-harness | ||
| run: uv pip install --system -e 'eval/.agent-eval-harness[anthropic]' | ||
| - name: Install yq | ||
| run: | | ||
| curl -sSfL "https://github.com/mikefarah/yq/releases/download/v4.47.1/yq_linux_amd64" -o /usr/local/bin/yq | ||
| echo "0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb /usr/local/bin/yq" | sha256sum -c | ||
| chmod +x /usr/local/bin/yq | ||
| - name: Configure git identity | ||
| run: | | ||
| git config --global user.name "fullsend-eval[bot]" | ||
| git config --global user.email "fullsend-eval[bot]@users.noreply.github.com" | ||
| - name: Validate fullsend ref | ||
| env: | ||
| FULLSEND_REF: ${{ inputs.fullsend_ref || 'main' }} | ||
| run: | | ||
| if [[ "${FULLSEND_REF}" != "main" && ! "${FULLSEND_REF}" =~ ^v[0-9] ]]; then | ||
| echo "::error::fullsend_ref must be 'main' or a release tag (v*), got '${FULLSEND_REF}'" | ||
| exit 1 | ||
| fi | ||
| - name: Clone fullsend | ||
| env: | ||
| GH_TOKEN: ${{ github.token }} | ||
| FULLSEND_REF: ${{ inputs.fullsend_ref || 'main' }} | ||
| run: | | ||
| FULLSEND_CHECKOUT="${RUNNER_TEMP}/fullsend" | ||
| git clone --depth 1 --branch "${FULLSEND_REF}" \ | ||
| "https://x-access-token:${GH_TOKEN}@github.com/fullsend-ai/fullsend.git" \ | ||
| "${FULLSEND_CHECKOUT}" | ||
| echo "FULLSEND_CHECKOUT=${FULLSEND_CHECKOUT}" >> "$GITHUB_ENV" | ||
| - name: Setup Go | ||
| uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | ||
| with: | ||
| go-version-file: ${{ env.FULLSEND_CHECKOUT }}/go.mod | ||
| cache-dependency-path: ${{ env.FULLSEND_CHECKOUT }}/go.sum | ||
| - name: Build fullsend | ||
| run: | | ||
| make -C "${FULLSEND_CHECKOUT}" go-build | ||
| echo "${FULLSEND_CHECKOUT}/bin" >> "$GITHUB_PATH" | ||
| "${FULLSEND_CHECKOUT}/bin/fullsend" --version | ||
| - name: Install Podman | ||
| run: | | ||
| sudo apt-get update | ||
| sudo apt-get install -y podman | ||
| - name: Configure rootless Podman | ||
| run: | | ||
| whoami_user="$(whoami)" | ||
| grep -q "^${whoami_user}:" /etc/subuid || sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 "${whoami_user}" | ||
| podman system migrate | ||
| - name: Start Podman API service | ||
| run: | | ||
| # Socket-activated under the user's systemd, as in fullsend's | ||
| # action.yml. The OpenShell gateway is a systemd user service that | ||
| # pins the podman driver and needs this socket to answer when it | ||
| # starts. With a `podman system service &` started from a step | ||
| # instead, it failed with "no responsive Podman API socket found". | ||
| systemctl --user start podman.socket | ||
| SOCKET_PATH="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock" | ||
| for _i in $(seq 1 30); do | ||
| podman --url "unix://${SOCKET_PATH}" info >/dev/null 2>&1 && break | ||
| sleep 1 | ||
| done | ||
| if ! podman --url "unix://${SOCKET_PATH}" info >/dev/null 2>&1; then | ||
| echo "::error::Podman socket not ready at ${SOCKET_PATH}" | ||
| systemctl --user status podman.socket podman.service --no-pager || true | ||
| journalctl --user -u podman.socket -u podman.service --no-pager -n 40 || true | ||
| exit 1 | ||
| fi | ||
| # After Podman: the gateway pins compute_driver = "podman" and does not | ||
| # start without the Podman API socket. | ||
| - name: Configure OpenShell gateway | ||
| run: | | ||
| eval "$(grep -E '^OPENSHELL_(VERSION|SHA)=' "${FULLSEND_CHECKOUT}/.github/scripts/openshell-version.sh")" | ||
| if [ -z "$OPENSHELL_VERSION" ] || [ -z "$OPENSHELL_SHA" ]; then | ||
| echo "::error::Failed to resolve OpenShell version from fullsend checkout" | ||
| exit 1 | ||
| fi | ||
| echo "OPENSHELL_VERSION=${OPENSHELL_VERSION}" >> "$GITHUB_ENV" | ||
| echo "OPENSHELL_SHA=${OPENSHELL_SHA}" >> "$GITHUB_ENV" | ||
| mkdir -p "$HOME/.config/openshell" | ||
| echo "OPENSHELL_BIND_ADDRESS=0.0.0.0" > "$HOME/.config/openshell/gateway.env" | ||
| cat > "$HOME/.config/openshell/gateway.toml" << EOF | ||
| [openshell] | ||
| version = 2 | ||
| [openshell.gateway] | ||
| compute_driver = "podman" | ||
| [openshell.drivers.podman] | ||
| supervisor_image = "ghcr.io/nvidia/openshell/supervisor:${OPENSHELL_VERSION}" | ||
| health_check_interval_secs = 10 | ||
| EOF | ||
| - name: Install OpenShell CLI | ||
| run: | | ||
| "${FULLSEND_CHECKOUT}/.github/scripts/install-openshell.sh" | ||
| - name: Install validation dependencies | ||
| run: pip install --quiet "jsonschema>=4.18.0" | ||
| - name: Check for secrets | ||
| id: secrets-check | ||
| run: | | ||
| if [ -z "$WIF_PROVIDER" ]; then | ||
| echo "::warning::GCP secrets are not configured. Skipping functional tests." | ||
| echo "available=false" >> "$GITHUB_OUTPUT" | ||
| else | ||
| echo "available=true" >> "$GITHUB_OUTPUT" | ||
| fi | ||
| env: | ||
| WIF_PROVIDER: ${{ secrets.E2E_GCP_WIF_PROVIDER }} | ||
| - name: Authenticate to GCP | ||
| if: steps.secrets-check.outputs.available == 'true' | ||
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 | ||
| with: | ||
| workload_identity_provider: ${{ secrets.E2E_GCP_WIF_PROVIDER }} | ||
| service_account: ${{ secrets.E2E_GCP_SERVICE_ACCOUNT }} | ||
| - name: Prepare sandbox credentials | ||
| if: steps.secrets-check.outputs.available == 'true' | ||
| run: | | ||
| echo "HOST_GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS" >> "$GITHUB_ENV" | ||
| if [[ -f scripts/prepare-sandbox-credentials.sh ]]; then | ||
| bash scripts/prepare-sandbox-credentials.sh | ||
| fi | ||
| - name: Run functional tests | ||
| if: steps.secrets-check.outputs.available == 'true' | ||
| env: | ||
| EVAL_ORG: ${{ vars.EVAL_ORG }} | ||
| GH_TOKEN: ${{ secrets.EVAL_GH_TOKEN }} | ||
| ANTHROPIC_VERTEX_PROJECT_ID: ${{ vars.EVALS_VERTEX_PROJECT_ID }} | ||
| GOOGLE_CLOUD_PROJECT: ${{ secrets.E2E_GCP_PROJECT_ID }} | ||
| CLOUD_ML_REGION: ${{ vars.EVALS_GCP_REGION }} | ||
| EVALS_HOST_CREDENTIALS: ${{ env.HOST_GOOGLE_APPLICATION_CREDENTIALS }} | ||
| FULLSEND_DIR: ${{ github.workspace }} | ||
| AGENT: ${{ matrix.agent }} | ||
| # An explicit tier input (dispatch, workflow_call) wins; a | ||
| # workflow_call sees its caller's event_name, so the input must | ||
| # be checked first. Otherwise merge_group runs full, and | ||
| # pull_request_target runs release, or full with the eval-full | ||
| # label. Anything else leaves EVAL_TIER empty, i.e. the script's | ||
| # default. | ||
| EVAL_TIER: >- | ||
| ${{ (inputs.tier != '' && inputs.tier != 'auto') && inputs.tier | ||
| || github.event_name == 'merge_group' && 'full' | ||
| || github.event_name == 'pull_request_target' | ||
| && (contains(github.event.pull_request.labels.*.name, 'eval-full') && 'full' || 'release') | ||
| || '' }} | ||
| run: ./eval/run-functional.sh "$AGENT" | ||
| # Protected-path change (human review required): scrub eval artifacts before | ||
| # upload so ::add-mask:: values (EVAL_GH_TOKEN) are not archived. See PR #184. | ||
| - name: Scrub secrets from eval results | ||
| id: scrub-eval-results | ||
| if: always() && steps.secrets-check.outputs.available == 'true' | ||
| run: bash eval/scripts/scrub-eval-results.sh eval/runs /tmp/agent-eval | ||
| # Upload only when scrub succeeded — never publish if live secrets remain. | ||
| - name: Upload eval results | ||
| if: >- | ||
| always() | ||
| && steps.secrets-check.outputs.available == 'true' | ||
| && steps.scrub-eval-results.outcome == 'success' | ||
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | ||
| with: | ||
| name: eval-results-${{ matrix.agent }} | ||
| path: | | ||
| eval/runs/ | ||
| !eval/runs/**/.eval-env | ||
| retention-days: 30 | ||
| # --------------------------------------------------------------------------- | ||
| # Roll-up job with a stable name for branch protection required checks. | ||
| # Require "functional-tests-complete" in repo settings — it handles all | ||
| # dynamic matrix shapes including empty (no agents to test). | ||
| # --------------------------------------------------------------------------- | ||
| functional-tests-complete: | ||
| needs: [gate, detect, functional-tests] | ||
| # Always runs. A pull_request_target `labeled` event for a label other | ||
| # than ok-to-test is handled inside check-rollup-result.sh, which mirrors | ||
| # the previous verdict on this commit rather than asserting a new one. | ||
| # Guarding this job with the gate's own `if:` was tried and rejected: a | ||
| # skipped job still publishes a check run that supersedes the previous | ||
| # one, and `skipped` satisfies a required check — so it would turn a | ||
| # genuinely failing run green the moment anyone labelled the PR. See #954. | ||
| if: always() | ||
| runs-on: ubuntu-24.04 | ||
| timeout-minutes: 1 | ||
| permissions: | ||
| contents: read | ||
| # Read-only: the roll-up mirrors the previous check run of its own name | ||
| # on this commit when a label event carries no verdict. | ||
| checks: read | ||
| steps: | ||
| # Base branch only -- a PR must not be able to rewrite this script | ||
| # to force-pass its own roll-up check. | ||
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | ||
| with: | ||
| repository: ${{ github.repository != 'fullsend-ai/agents' && 'fullsend-ai/agents' || github.repository }} | ||
| ref: >- | ||
| ${{ github.repository != 'fullsend-ai/agents' | ||
| && (needs.detect.outputs.agents_sha || 'main') | ||
| || github.sha }} | ||
| persist-credentials: false | ||
| sparse-checkout: .github/scripts | ||
| - name: Check results | ||
| env: | ||
| EVENT_NAME: ${{ github.event_name }} | ||
| EVENT_ACTION: ${{ github.event.action }} | ||
| LABEL_NAME: ${{ github.event.label.name }} | ||
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | ||
| GATE_RESULT: ${{ needs.gate.result }} | ||
| DETECT_RESULT: ${{ needs.detect.result }} | ||
| TESTS_RESULT: ${{ needs.functional-tests.result }} | ||
| CROSS_REPO: ${{ github.repository != 'fullsend-ai/agents' }} | ||
| GH_TOKEN: ${{ github.token }} | ||
| run: bash .github/scripts/check-rollup-result.sh | ||
