midscene-omarchy-4.0.3.yml

name: Omarchy 4.0.3
run-name: Omarchy 4.0.3

on:
  push:
    branches: [main]
    paths: &e2e-paths
      - ".github/workflows/midscene-omarchy-4.0.3.yml"
      - ".github/workflows/deploy-midscene-report.yml"
      - "scripts/build-pages-report.mjs"
      - "scripts/report-cases.mjs"
      - "scripts/render-ci-report-summary.mjs"
      - "scripts/write-ci-shard-status.mjs"
      - "scripts/verify-pages-report.mjs"
      - "scripts/verify-omarchy-evidence.mjs"
      - "scripts/omarchy-shell-evidence.mjs"
      - "scripts/parse-omarchy-review-issue.mjs"
      - "tests/contracts/test_pages_report_history.mjs"
      - "tests/contracts/test_model_rate_gate.mjs"
      - "tests/contracts/test_omarchy_review_issue.mjs"
      - "tests/e2e/package.json"
      - "tests/e2e/package-lock.json"
      - "tests/e2e/gtk_fixture.py"
      - "tests/e2e/gtk_onboarding_fixture.py"
      - "tests/e2e/gtk_runtime_fixture.py"
      - "tests/e2e/capture-report-preview.mjs"
      - "tests/e2e/cases/**"
      - "tests/e2e/midscene.config.ts"
      - "tests/e2e/model-rate-gate.mjs"
      - "tests/e2e/render-omarchy-plugin-smoke.mjs"
      - "tests/e2e/run-omarchy-midscene.sh"
      - "tests/e2e/restore-omarchy-vm.sh"
      - "tests/e2e/omarchy-vm.env"
      - "src/**"
      - "manifest.json"
      - "install.sh"
      - "setup-omarchy.sh"
      - "start.sh"
      - "omarchy/**"
  pull_request:
    branches: [main]
    paths: *e2e-paths
  issues:
    types: [labeled]
  workflow_dispatch:
    inputs:
      bootstrap_hosted_model:
        description: Transfer the existing hosted model secret to the Midscene App
        type: boolean
        required: false
        default: false
      project:
        description: Run one Midscene project for focused verification
        type: choice
        default: all
        options:
          - all
          - omarchy-shard-1
          - omarchy-shard-2
          - omarchy-shard-3
          - omarchy-shard-4
          - omarchy-shell
          - omarchy-plugin-review
          - omarchy-plugin-smoke
          - omarchy-providers
      plugin_repository:
        description: Public GitHub owner/repository for omarchy-plugin-smoke
        type: string
        required: false
      plugin_sha:
        description: Exact 40-character commit SHA for omarchy-plugin-smoke
        type: string
        required: false
      plugin_id:
        description: Plugin manifest ID for omarchy-plugin-smoke
        type: string
        required: false
      plugin_open_method:
        description: Use summon for panel/overlay/menu, or a plugin IPC method such as open
        type: string
        required: false
        default: open
      visible_assertion:
        description: What should be visible after opening the plugin
        type: string
        required: false
      review_run_id:
        description: Midscene GitHub App review run ID
        type: string
        required: false

permissions:
  contents: read
  packages: read
  id-token: write

concurrency:
  group: midscene-model-e2e-${{ github.workflow }}-${{ github.event.issue.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  bootstrap-hosted-model:
    name: Bootstrap hosted model
    if: github.event_name == 'workflow_dispatch' && inputs.bootstrap_hosted_model == true && github.actor == 'quanru' && github.ref == 'refs/heads/research/omarchy-plugin-visual-review'
    runs-on: ubuntu-latest
    timeout-minutes: 5
    env:
      MIDSCENE_REVIEW_APP_URL: ${{ vars.MIDSCENE_REVIEW_APP_URL }}
      HOSTED_MODEL_API_KEY: ${{ secrets.MIDSCENE_MODEL_API_KEY }}
      HOSTED_MODEL_NAME: ${{ secrets.MIDSCENE_MODEL_NAME }}
      HOSTED_MODEL_BASE_URL: ${{ secrets.MIDSCENE_MODEL_BASE_URL }}
      HOSTED_MODEL_FAMILY: ${{ secrets.MIDSCENE_MODEL_FAMILY }}
    steps:
      - name: Transfer hosted model configuration over OIDC-authenticated HTTPS
        run: |
          test -n "$MIDSCENE_REVIEW_APP_URL"
          test -n "$HOSTED_MODEL_API_KEY"
          test -n "$HOSTED_MODEL_NAME"
          test -n "$HOSTED_MODEL_BASE_URL"
          test -n "$HOSTED_MODEL_FAMILY"
          node --input-type=module <<'JS'
          const identityResponse = await fetch(`${process.env.ACTIONS_ID_TOKEN_REQUEST_URL}&audience=midscene-review-app`, {
            headers: { authorization: `bearer ${process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN}` },
          });
          if (!identityResponse.ok) throw new Error(`GitHub OIDC failed: ${identityResponse.status}`);
          const { value: identity } = await identityResponse.json();
          const response = await fetch(`${process.env.MIDSCENE_REVIEW_APP_URL.replace(/\/+$/, '')}/api/worker/bootstrap-hosted`, {
            method: 'POST',
            headers: { authorization: `Bearer ${identity}`, 'content-type': 'application/json' },
            body: JSON.stringify({
              apiKey: process.env.HOSTED_MODEL_API_KEY,
              modelName: process.env.HOSTED_MODEL_NAME,
              baseUrl: process.env.HOSTED_MODEL_BASE_URL,
              modelFamily: process.env.HOSTED_MODEL_FAMILY,
            }),
          });
          if (!response.ok) throw new Error(`Midscene bootstrap failed: ${response.status}`);
          console.log('Hosted model configuration was stored by the Midscene review service.');
          JS

  omarchy-e2e:
    name: Midscene visual (${{ matrix.project }})
    if: >-
      inputs.bootstrap_hosted_model != true &&
      (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
      (github.event_name != 'issues' ||
      (github.event.label.name == 'midscene-review' && startsWith(github.event.issue.title, '[Omarchy review] ')))
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      # Each shard restores an independent disposable VM. Sharing one Hyprland
      # session would make fixture resets, focus and screenshots race.
      # Run independent VMs concurrently. Each worker paces model requests to
      # one every 20s, keeping the aggregate below the Ark endpoint RPM.
      fail-fast: false
      max-parallel: 5
      matrix:
        project: >-
          ${{ fromJSON(github.event_name == 'issues' && '["omarchy-plugin-smoke"]' ||
          (inputs.project != '' && inputs.project != 'all') &&
          format('["{0}"]', inputs.project) ||
          '["omarchy-shard-1","omarchy-shard-2","omarchy-shard-3","omarchy-shard-4","omarchy-shell"]') }}
    env:
      MIDSCENE_COMPUTER_HEADLESS_LINUX: "true"
      MIDSCENE_REPLANNING_CYCLE_LIMIT: "40"
      # The Ark endpoint answers bursts with 429; retry up to 3 extra times
      # with a 65s fixed pause so the wait spans the 60s RPM window.
      MIDSCENE_MODEL_RETRY_COUNT: "3"
      MIDSCENE_MODEL_RETRY_INTERVAL: "65000"
      # The configured model endpoint is reachable over IPv4; disabling Node's
      # family race avoids intermittent runner-side IPv6 connection timeouts.
      NODE_OPTIONS: "--dns-result-order=ipv4first --no-network-family-autoselection"
      REVIEW_PLUGIN_REPOSITORY: ${{ inputs.plugin_repository }}
      REVIEW_PLUGIN_SHA: ${{ inputs.plugin_sha }}
      REVIEW_PLUGIN_ID: ${{ inputs.plugin_id }}
      REVIEW_PLUGIN_OPEN_METHOD: ${{ inputs.plugin_open_method }}
      REVIEW_VISIBLE_ASSERTION: ${{ inputs.visible_assertion }}
      MIDSCENE_REVIEW_APP_URL: ${{ vars.MIDSCENE_REVIEW_APP_URL }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Read maintainer-approved review Issue
        if: github.event_name == 'issues'
        run: node scripts/parse-omarchy-review-issue.mjs

      - name: Configure existing worker model
        if: github.event_name != 'workflow_dispatch' || inputs.review_run_id == ''
        env:
          LEGACY_MODEL_API_KEY: ${{ secrets.MIDSCENE_MODEL_API_KEY }}
          LEGACY_MODEL_NAME: ${{ secrets.MIDSCENE_MODEL_NAME }}
          LEGACY_MODEL_BASE_URL: ${{ secrets.MIDSCENE_MODEL_BASE_URL }}
          LEGACY_MODEL_FAMILY: ${{ secrets.MIDSCENE_MODEL_FAMILY }}
          LEGACY_MODEL_REASONING_ENABLED: ${{ secrets.MIDSCENE_MODEL_REASONING_ENABLED }}
        run: |
          {
            echo "MIDSCENE_MODEL_API_KEY=$LEGACY_MODEL_API_KEY"
            echo "MIDSCENE_MODEL_NAME=$LEGACY_MODEL_NAME"
            echo "MIDSCENE_MODEL_BASE_URL=$LEGACY_MODEL_BASE_URL"
            echo "MIDSCENE_MODEL_FAMILY=$LEGACY_MODEL_FAMILY"
            echo "MIDSCENE_MODEL_REASONING_ENABLED=$LEGACY_MODEL_REASONING_ENABLED"
          } >> "$GITHUB_ENV"

      - name: Obtain scoped Midscene model proxy token
        if: github.event_name == 'workflow_dispatch' && inputs.review_run_id != ''
        env:
          REVIEW_RUN_ID: ${{ inputs.review_run_id }}
        run: |
          test -n "$MIDSCENE_REVIEW_APP_URL"
          oidc=$(curl --fail --silent --show-error \
            -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
            "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=midscene-review-app" | jq -r '.value')
          response=$(curl --fail --silent --show-error \
            -H "Authorization: Bearer $oidc" \
            -H "Content-Type: application/json" \
            --data "$(jq -nc --arg id "$REVIEW_RUN_ID" '{runId:$id}')" \
            "${MIDSCENE_REVIEW_APP_URL%/}/api/worker/model-token")
          proxy_token=$(jq -er '.token' <<< "$response")
          echo "::add-mask::$proxy_token"
          {
            echo "MIDSCENE_MODEL_API_KEY=$proxy_token"
            echo "MIDSCENE_MODEL_NAME=$(jq -er '.modelName' <<< "$response")"
            echo "MIDSCENE_MODEL_FAMILY=$(jq -er '.modelFamily' <<< "$response")"
            echo "MIDSCENE_MODEL_BASE_URL=$(jq -er '.baseUrl' <<< "$response")"
          } >> "$GITHUB_ENV"

      - name: Require Midscene model configuration
        run: |
          test -n "$MIDSCENE_MODEL_API_KEY"
          test -n "$MIDSCENE_MODEL_NAME"
          test -n "$MIDSCENE_MODEL_BASE_URL"
          test -n "$MIDSCENE_MODEL_FAMILY"

      - name: Validate plugin review request
        if: matrix.project == 'omarchy-plugin-smoke'
        run: |
          [[ "$REVIEW_PLUGIN_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]
          [[ "$REVIEW_PLUGIN_SHA" =~ ^[a-fA-F0-9]{40}$ ]]
          [[ "$REVIEW_PLUGIN_ID" =~ ^[a-z0-9][a-z0-9._-]{2,127}$ ]]
          [[ "$REVIEW_PLUGIN_OPEN_METHOD" =~ ^[A-Za-z][A-Za-z0-9_]*$ ]]
          test -n "$REVIEW_VISIBLE_ASSERTION"
          test "${#REVIEW_VISIBLE_ASSERTION}" -le 500

      - name: Check Midscene model before the expensive VM install
        run: |
          payload=$(jq -nc --arg model "$MIDSCENE_MODEL_NAME" '{
            model: $model,
            messages: [{role: "user", content: "Reply OK."}],
            max_tokens: 1
          }')
          curl --fail --silent --show-error \
            --connect-timeout 15 --max-time 90 \
            --retry 4 --retry-all-errors --retry-delay 5 --retry-max-time 90 \
            -H "Authorization: Bearer $MIDSCENE_MODEL_API_KEY" \
            -H "Content-Type: application/json" \
            --data "$payload" \
            "${MIDSCENE_MODEL_BASE_URL%/}/chat/completions" \
            >/dev/null

      - name: Install ORAS client
        uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1

      - name: Sign in to GitHub Container Registry
        run: echo "$GHCR_TOKEN" | oras login ghcr.io --username "$GITHUB_ACTOR" --password-stdin
        env:
          GHCR_TOKEN: ${{ github.token }}

      - name: Restore prebuilt Omarchy VM
        run: tests/e2e/restore-omarchy-vm.sh

      - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
        with:
          node-version: "22"
          cache: npm
          cache-dependency-path: tests/e2e/package-lock.json

      - name: Install Midscene and VNC bridge
        id: test-dependencies
        run: |
          sudo apt-get install -y --no-install-recommends \
            fluxbox tigervnc-viewer x11-xserver-utils xvfb
          for attempt in 1 2 3; do
            if npm --prefix tests/e2e ci --include=optional --ignore-scripts; then
              exit 0
            fi
            echo "npm install attempt $attempt failed; retrying." >&2
            sleep 10
          done
          exit 1

      - name: Run real Omarchy Midscene project
        id: midscene-test
        env:
          PROJECT: ${{ matrix.project }}
        run: tests/e2e/run-omarchy-midscene.sh "$PROJECT"

      - name: Capture report evidence
        id: capture-report
        if: always() && !cancelled() && steps.test-dependencies.outcome == 'success'
        env:
          PROJECT: ${{ matrix.project }}
        run: |
          node tests/e2e/capture-report-preview.mjs \
            --report-dir tests/e2e/midscene_run \
            --projects "$PROJECT"

      - name: Record shard result for report aggregation
        if: always() && !cancelled()
        env:
          PROJECT: ${{ matrix.project }}
          TEST_OUTCOME: ${{ steps.midscene-test.outcome }}
          CAPTURE_OUTCOME: ${{ steps.capture-report.outcome }}
        run: |
          node scripts/write-ci-shard-status.mjs \
            --output "tests/e2e/midscene_run/ci-shard-status-$PROJECT.json" \
            --project "$PROJECT" \
            --test-outcome "${TEST_OUTCOME:-skipped}" \
            --capture-outcome "${CAPTURE_OUTCOME:-skipped}"

      - name: Upload installer evidence
        if: always() && !cancelled()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: omarchy-vm-installer-evidence-${{ matrix.project }}
          path: .midscene-omarchy/omarchy-iso/test-runs/**/runs/**
          if-no-files-found: warn
          retention-days: 3
          overwrite: true

      - name: Upload Omarchy Midscene shard report
        if: always() && !cancelled()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: omarchy-midscene-${{ matrix.project }}
          path: tests/e2e/midscene_run/
          if-no-files-found: error
          retention-days: 7
          overwrite: true

      - name: Publish direct HTML report to Midscene
        id: publish-report
        if: always() && !cancelled() && github.event_name == 'workflow_dispatch' && inputs.review_run_id != ''
        continue-on-error: true
        env:
          REVIEW_RUN_ID: ${{ inputs.review_run_id }}
        run: |
          oidc=$(curl --fail --silent --show-error \
            -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
            "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=midscene-review-app" | jq -r '.value')
          report_root=$(find tests/e2e/midscene_run/report -mindepth 1 -maxdepth 1 -type d -name 'midscene-e2e-*' -print -quit)
          test -f "$report_root/index.html"
          while IFS= read -r -d '' file; do
            relative=${file#"$report_root"/}
            if grep -Fq -- "$MIDSCENE_MODEL_API_KEY" "$file"; then
              echo "Refusing to publish a report containing the model proxy token." >&2
              exit 1
            fi
            curl --fail --silent --show-error \
              -X PUT -H "Authorization: Bearer $oidc" \
              --data-binary @"$file" \
              "${MIDSCENE_REVIEW_APP_URL%/}/api/worker/report/$REVIEW_RUN_ID/$relative" >/dev/null
          done < <(find "$report_root" -type f -print0)

      - name: Complete Midscene review check
        if: always() && !cancelled() && github.event_name == 'workflow_dispatch' && inputs.review_run_id != ''
        env:
          REVIEW_RUN_ID: ${{ inputs.review_run_id }}
          TEST_OUTCOME: ${{ steps.midscene-test.outcome }}
          REPORT_OUTCOME: ${{ steps.publish-report.outcome }}
        run: |
          if [ "$REPORT_OUTCOME" = success ]; then sleep 65; fi
          oidc=$(curl --fail --silent --show-error \
            -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
            "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=midscene-review-app" | jq -r '.value')
          outcome=failed
          if [ "$TEST_OUTCOME" = success ] && [ "$REPORT_OUTCOME" = success ]; then outcome=succeeded; fi
          curl --fail --silent --show-error \
            -H "Authorization: Bearer $oidc" \
            -H "Content-Type: application/json" \
            --data "$(jq -nc --arg id "$REVIEW_RUN_ID" --arg status "$outcome" '{runId:$id,outcome:$status}')" \
            "${MIDSCENE_REVIEW_APP_URL%/}/api/worker/complete" >/dev/null

      - name: Verify Omarchy shell visual evidence
        if: success() && matrix.project == 'omarchy-shell'
        run: node scripts/verify-omarchy-evidence.mjs tests/e2e/midscene_run

  report-bundle:
    name: Assemble Midscene report bundle
    needs: omarchy-e2e
    if: >-
      always() && !cancelled() && github.event_name != 'issues' && inputs.bootstrap_hosted_model != true &&
      (inputs.project == '' || inputs.project == 'all')
    runs-on: ubuntu-latest
    outputs:
      report-artifact-id: ${{ steps.report.outputs.artifact-id }}
    steps:
      - name: Create bundle even when every shard failed early
        run: |
          mkdir -p "$RUNNER_TEMP/midscene-report-shards"
          printf '%s\n' '{"bundle":"omarchy-4.0.3"}' > "$RUNNER_TEMP/midscene-report-shards/ci-bundle-status.json"

      # Keep shard directories separate. Native report filenames can share a
      # timestamp, while the Pages builder locates reports by project identity.
      - name: Download every shard report
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          # Exclude the previous attempt's combined omarchy-midscene-e2e-report
          # artifact when rerunning failed shards of the same workflow run.
          pattern: omarchy-midscene-omarchy-*
          path: ${{ runner.temp }}/midscene-report-shards
          merge-multiple: false

      - name: Upload combined report bundle
        id: report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: omarchy-midscene-e2e-report
          path: ${{ runner.temp }}/midscene-report-shards/
          if-no-files-found: error
          retention-days: 7
          overwrite: true

  pages-report:
    name: GitHub Pages report
    needs: [omarchy-e2e, report-bundle]
    if: >-
      always() && github.event_name != 'issues' && inputs.bootstrap_hosted_model != true &&
      !cancelled() &&
      needs.report-bundle.outputs.report-artifact-id != '' &&
      (github.event_name != 'pull_request' ||
      github.event.pull_request.head.repo.full_name == github.repository)
    permissions:
      actions: read
      contents: read
      pages: write
      id-token: write
    uses: ./.github/workflows/deploy-midscene-report.yml
    with:
      artifact-name: omarchy-midscene-e2e-report
      report-label: Omarchy 4.0.3
      summary-title: Omarchy
      report-groups: >-
        [{"role":"primary","label":"Doubao Say","projects":["omarchy-shard-1","omarchy-shard-2","omarchy-shard-3","omarchy-shard-4"]},{"role":"auxiliary","label":"Omarchy visual checks","projects":["omarchy-shell"]}]
      run-result: ${{ needs.omarchy-e2e.result }}

  review-issue-result:
    name: Link visual evidence to review request
    needs: omarchy-e2e
    if: >-
      always() && github.event_name == 'issues' &&
      github.event.label.name == 'midscene-review' &&
      startsWith(github.event.issue.title, '[Omarchy review] ')
    runs-on: ubuntu-latest
    permissions:
      issues: write
    env:
      GH_TOKEN: ${{ github.token }}
      ISSUE_NUMBER: ${{ github.event.issue.number }}
      REVIEW_RESULT: ${{ needs.omarchy-e2e.result }}
    steps:
      - name: Comment with CI run and artifact
        run: |
          cat > "$RUNNER_TEMP/omarchy-review-comment.md" <<EOF
          Midscene Omarchy visual review CI: **${REVIEW_RESULT}**.

          [Open the GitHub Actions run](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) and download the omarchy-midscene-omarchy-plugin-smoke artifact for the native report and screenshots.

          This is behavioral evidence from a disposable VM. It does not change the Marketplace security baseline or Verified status. A failed run needs its logs reviewed before attributing the failure to the plugin.
          EOF
          gh issue comment "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" --body-file "$RUNNER_TEMP/omarchy-review-comment.md"