diff --git a/.github/workflows/comment-ops.yml b/.github/workflows/comment-ops.yml index 008b453a94..65e5583860 100644 --- a/.github/workflows/comment-ops.yml +++ b/.github/workflows/comment-ops.yml @@ -28,26 +28,34 @@ jobs: is_coder: ${{ steps.route.outputs.is_coder || 'false' }} is_review_issues: ${{ steps.route.outputs.is_review_issues || 'false' }} is_enrich: ${{ steps.route.outputs.is_enrich || 'false' }} + is_ui_review: ${{ steps.route.outputs.is_ui_review || 'false' }} is_help: ${{ steps.route.outputs.is_help || 'false' }} steps: - id: route env: BODY: ${{ github.event.comment.body }} run: | - # Use case-insensitive grep and ensure outputs are always set if matched. - # We check for commands starting at the beginning of a line to avoid accidental triggers - # from quotes or descriptive text mentioning the bots. - if echo "$BODY" | grep -qiE "(^|\r?\n)@(gemini-bot|gemini-review)"; then echo "is_review=true" >> $GITHUB_OUTPUT; fi - if echo "$BODY" | grep -qiE "(^|\r?\n)@pr-squash"; then echo "is_squash=true" >> $GITHUB_OUTPUT; fi - if echo "$BODY" | grep -qiE "(^|\r?\n)@conflict-resolve"; then echo "is_resolve=true" >> $GITHUB_OUTPUT; fi - if echo "$BODY" | grep -qiE "^@gemini-triage"; then echo "is_triage=true" >> $GITHUB_OUTPUT; fi - if echo "$BODY" | grep -qiE "^@gemini-coder"; then echo "is_coder=true" >> $GITHUB_OUTPUT; fi - if echo "$BODY" | grep -qiE "^@create-review-issues"; then echo "is_review_issues=true" >> $GITHUB_OUTPUT; fi - if echo "$BODY" | grep -qiE "^@gemini-enrich"; then echo "is_enrich=true" >> $GITHUB_OUTPUT; fi - if echo "$BODY" | grep -qiE "^@gemini-help"; then echo "is_help=true" >> $GITHUB_OUTPUT; fi - - # Set consolidated trigger flag for status jobs - if echo "$BODY" | grep -qiE "(^|\r?\n)@(gemini-bot|gemini-review|pr-squash|conflict-resolve|gemini-triage|gemini-coder|create-review-issues|gemini-enrich|gemini-help)"; then + COMMANDS=( + "(gemini-bot|gemini-review):review" + "pr-squash:squash" + "conflict-resolve:resolve" + "gemini-triage:triage" + "gemini-coder:coder" + "create-review-issues:review_issues" + "gemini-enrich:enrich" + "gemini-ui-review:ui_review" + "gemini-help:help" + ) + + for cmd in "${COMMANDS[@]}"; do + KEY=${cmd%%:*} + VAR=${cmd#*:} + if echo "$BODY" | grep -qiE "(^|\r?\n)@$KEY"; then + echo "is_$VAR=true" >> $GITHUB_OUTPUT + fi + done + + if grep -q "true" "$GITHUB_OUTPUT"; then echo "is_command_triggered=true" >> $GITHUB_OUTPUT fi @@ -100,6 +108,12 @@ jobs: manual_trigger: true secrets: inherit + execute-ui-review: + needs: [router] + if: github.event.issue.pull_request && needs.router.outputs.is_ui_review == 'true' + uses: ./.github/workflows/gemini-ui-review.yml + secrets: inherit + prepare-coder: needs: [router] if: needs.router.outputs.is_coder == 'true' @@ -225,6 +239,7 @@ jobs: '| :--- | :--- |\n' + '| `@gemini-bot` | Run AI Code Review (PR only) |\n' + '| `@gemini-enrich` | Run PR Enrichment (PR only) |\n' + + '| `@gemini-ui-review` | Run Multimodal UI Review (PR only) |\n' + '| `@gemini-triage` | Run Issue Triage |\n' + '| `@gemini-coder ` | Generate Code |\n' + '| `@create-review-issues` | Create issues from review (PR only) |\n' + @@ -301,6 +316,7 @@ jobs: - init-status - execute-review - execute-triage + - execute-ui-review - execute-coder - execute-create-review-issues - execute-help @@ -331,6 +347,7 @@ jobs: check_job "Review" "${{ needs.execute-review.result }}" "${{ needs.router.outputs.is_review }}" check_job "Triage" "${{ needs.execute-triage.result }}" "${{ needs.router.outputs.is_triage }}" + check_job "UI Review" "${{ needs.execute-ui-review.result }}" "${{ needs.router.outputs.is_ui_review }}" check_job "Coder" "${{ needs.execute-coder.result }}" "${{ needs.router.outputs.is_coder }}" check_job "Create Review Issues (Manual)" "${{ needs.execute-create-review-issues.result }}" "${{ needs.router.outputs.is_review_issues }}" diff --git a/.github/workflows/gemini-ui-review.yml b/.github/workflows/gemini-ui-review.yml new file mode 100644 index 0000000000..0a8b5ea467 --- /dev/null +++ b/.github/workflows/gemini-ui-review.yml @@ -0,0 +1,51 @@ +name: Gemini UI Review + +on: + workflow_call: + issue_comment: + types: [created] + +jobs: + ui-review: + if: | + (github.event_name == 'workflow_call') || + (github.event.issue.pull_request && contains(github.event.comment.body, '@gemini-ui-review')) + runs-on: ubuntu-latest + permissions: + pull-requests: write + contents: read + issues: write + steps: + - uses: actions/checkout@v4 + + - name: Setup Environment + uses: ./.github/actions/setup-env + + - name: Install Playwright Browsers + run: npx playwright install --with-deps chromium + + - name: Get Deployment URL + id: deploy-url + run: | + PR_NUMBER=${{ github.event.issue.number || github.event.pull_request.number }} + BRANCH=$(gh pr view "$PR_NUMBER" --json headRefName --jq '.headRefName') + + # Try to find target_url from the most recent deployment status for this branch + URL=$(gh api repos/${{ github.repository }}/deployments -q '.[] | select(.environment=="preview" and .ref=="'"$BRANCH"'") | .id' | head -n 1 | xargs -I {} gh api repos/${{ github.repository }}/deployments/{}/statuses -q 'map(select(.state == "success")) | .[0].target_url' 2>/dev/null || true) + + if [ -z "$URL" ] || [ "$URL" == "null" ]; then + echo "::error::No successful deployment found for branch $BRANCH. UI Review requires a preview environment." + exit 1 + fi + + echo "url=$URL" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Run UI Review Script + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.issue.number || github.event.pull_request.number }} + DEPLOYMENT_URL: ${{ steps.deploy-url.outputs.url }} + run: npx tsx scripts/ci/run-ui-review.ts diff --git a/package.json b/package.json index 374f59980f..a3fdfcb0d4 100644 --- a/package.json +++ b/package.json @@ -104,6 +104,7 @@ "@google/generative-ai": "0.24.1", "@jest/globals": "^30.2.0", "@next/bundle-analyzer": "^16.2.1", + "@octokit/rest": "^22.0.1", "@playwright/test": "^1.57.0", "@storybook/addon-a11y": "^10.1.11", "@storybook/addon-actions": "^8.6.14", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 82b1d26358..15a157ff48 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -132,6 +132,9 @@ importers: '@next/bundle-analyzer': specifier: ^16.2.1 version: 16.2.1 + '@octokit/rest': + specifier: ^22.0.1 + version: 22.0.1 '@playwright/test': specifier: ^1.57.0 version: 1.57.0 @@ -2257,6 +2260,58 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} + '@octokit/auth-token@6.0.0': + resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==} + engines: {node: '>= 20'} + + '@octokit/core@7.0.6': + resolution: {integrity: sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==} + engines: {node: '>= 20'} + + '@octokit/endpoint@11.0.3': + resolution: {integrity: sha512-FWFlNxghg4HrXkD3ifYbS/IdL/mDHjh9QcsNyhQjN8dplUoZbejsdpmuqdA76nxj2xoWPs7p8uX2SNr9rYu0Ag==} + engines: {node: '>= 20'} + + '@octokit/graphql@9.0.3': + resolution: {integrity: sha512-grAEuupr/C1rALFnXTv6ZQhFuL1D8G5y8CN04RgrO4FIPMrtm+mcZzFG7dcBm+nq+1ppNixu+Jd78aeJOYxlGA==} + engines: {node: '>= 20'} + + '@octokit/openapi-types@27.0.0': + resolution: {integrity: sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==} + + '@octokit/plugin-paginate-rest@14.0.0': + resolution: {integrity: sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-request-log@6.0.0': + resolution: {integrity: sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-rest-endpoint-methods@17.0.0': + resolution: {integrity: sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/request-error@7.1.0': + resolution: {integrity: sha512-KMQIfq5sOPpkQYajXHwnhjCC0slzCNScLHs9JafXc4RAJI+9f+jNDlBNaIMTvazOPLgb4BnlhGJOTbnN0wIjPw==} + engines: {node: '>= 20'} + + '@octokit/request@10.0.8': + resolution: {integrity: sha512-SJZNwY9pur9Agf7l87ywFi14W+Hd9Jg6Ifivsd33+/bGUQIjNujdFiXII2/qSlN2ybqUHfp5xpekMEjIBTjlSw==} + engines: {node: '>= 20'} + + '@octokit/rest@22.0.1': + resolution: {integrity: sha512-Jzbhzl3CEexhnivb1iQ0KJ7s5vvjMWcmRtq5aUsKmKDrRW6z3r84ngmiFKFvpZjpiU/9/S6ITPFRpn5s/3uQJw==} + engines: {node: '>= 20'} + + '@octokit/types@16.0.0': + resolution: {integrity: sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==} + '@open-draft/deferred-promise@2.2.0': resolution: {integrity: sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==} @@ -3531,6 +3586,9 @@ packages: resolution: {integrity: sha512-giSYKGTOcPZyJDbfbTtzAedLcNWdjCLbXYU3/MwPnjyvDXzu6Dgw8d2M+8jHhZXSmsCMSQqCp+YBsJ603UO4vQ==} hasBin: true + before-after-hook@4.0.0: + resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==} + better-opn@3.0.2: resolution: {integrity: sha512-aVNobHnJqLiUelTaHat9DZ1qM2w0C0Eym4LPI/3JxOnSokGVdsl1T1kN7TFvsEAD8G47A6VKQ0TVHqbBnYMJlQ==} engines: {node: '>=12.0.0'} @@ -4589,6 +4647,9 @@ packages: resolution: {integrity: sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==} engines: {node: '>=18'} + fast-content-type-parse@3.0.0: + resolution: {integrity: sha512-ZvLdcY8P+N8mGQJahJV5G4U88CSvT1rP8ApL6uETe88MBXrBHAkZlSEySdUlyztF7ccb+Znos3TFqaepHxdhBg==} + fast-copy@4.0.2: resolution: {integrity: sha512-ybA6PDXIXOXivLJK/z9e+Otk7ve13I4ckBvGO5I2RRmBU1gMHLVDJYEuJYhGwez7YNlYji2M2DvVU+a9mSFDlw==} @@ -5549,6 +5610,9 @@ packages: json-stringify-safe@5.0.1: resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + json-with-bigint@3.5.8: + resolution: {integrity: sha512-eq/4KP6K34kwa7TcFdtvnftvHCD9KvHOGGICWwMFc4dOOKF5t4iYqnfLK8otCRCRv06FXOzGGyqE8h8ElMvvdw==} + json5@1.0.2: resolution: {integrity: sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==} hasBin: true @@ -7404,6 +7468,9 @@ packages: resolution: {integrity: sha512-hpbDzxUY9BFwX+UeBnxv3Sh1q7HFxj48DTmXchNgRa46lO8uj3/1iEn3MiNUYTg1g9ctIqXCCERn8gYZhHC5lQ==} engines: {node: '>=4'} + universal-user-agent@7.0.3: + resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==} + universalify@2.0.1: resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} engines: {node: '>= 10.0.0'} @@ -9864,6 +9931,69 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} + '@octokit/auth-token@6.0.0': {} + + '@octokit/core@7.0.6': + dependencies: + '@octokit/auth-token': 6.0.0 + '@octokit/graphql': 9.0.3 + '@octokit/request': 10.0.8 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + before-after-hook: 4.0.0 + universal-user-agent: 7.0.3 + + '@octokit/endpoint@11.0.3': + dependencies: + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/graphql@9.0.3': + dependencies: + '@octokit/request': 10.0.8 + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/openapi-types@27.0.0': {} + + '@octokit/plugin-paginate-rest@14.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/types': 16.0.0 + + '@octokit/plugin-request-log@6.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + + '@octokit/plugin-rest-endpoint-methods@17.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/types': 16.0.0 + + '@octokit/request-error@7.1.0': + dependencies: + '@octokit/types': 16.0.0 + + '@octokit/request@10.0.8': + dependencies: + '@octokit/endpoint': 11.0.3 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + fast-content-type-parse: 3.0.0 + json-with-bigint: 3.5.8 + universal-user-agent: 7.0.3 + + '@octokit/rest@22.0.1': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/plugin-paginate-rest': 14.0.0(@octokit/core@7.0.6) + '@octokit/plugin-request-log': 6.0.0(@octokit/core@7.0.6) + '@octokit/plugin-rest-endpoint-methods': 17.0.0(@octokit/core@7.0.6) + + '@octokit/types@16.0.0': + dependencies: + '@octokit/openapi-types': 27.0.0 + '@open-draft/deferred-promise@2.2.0': {} '@open-draft/logger@0.3.0': @@ -11378,6 +11508,8 @@ snapshots: bats@1.13.0: {} + before-after-hook@4.0.0: {} + better-opn@3.0.2: dependencies: open: 8.4.2 @@ -12663,6 +12795,8 @@ snapshots: fake-indexeddb@6.2.5: {} + fast-content-type-parse@3.0.0: {} + fast-copy@4.0.2: {} fast-deep-equal@3.1.3: {} @@ -13858,6 +13992,8 @@ snapshots: json-stringify-safe@5.0.1: optional: true + json-with-bigint@3.5.8: {} + json5@1.0.2: dependencies: minimist: 1.2.8 @@ -15947,6 +16083,8 @@ snapshots: unicode-property-aliases-ecmascript@2.2.0: {} + universal-user-agent@7.0.3: {} + universalify@2.0.1: {} unpipe@1.0.0: {} diff --git a/scripts/ci/run-ui-review.ts b/scripts/ci/run-ui-review.ts new file mode 100644 index 0000000000..c87b373942 --- /dev/null +++ b/scripts/ci/run-ui-review.ts @@ -0,0 +1,186 @@ +import { chromium } from '@playwright/test' +import { GoogleGenerativeAI } from '@google/generative-ai' +import { Octokit } from '@octokit/rest' +import type { ServerMessage } from '../../types/websocket' + +const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY!) + +async function performUIReview() { + const browser = await chromium.launch() + const context = await browser.newContext({ + viewport: { width: 1280, height: 720 }, + }) + + const baseUrl = process.env.DEPLOYMENT_URL || 'http://localhost:3000' + + await context.addCookies([ + { name: 'session-id', value: 'mock', url: baseUrl }, + ]) + + const page = await context.newPage() + const targetUrl = new URL(baseUrl) + targetUrl.searchParams.set('testing', 'true') + + try { + await page.goto(targetUrl.toString(), { waitUntil: 'networkidle' }) + + await page.waitForSelector('[data-testid="main-content-layout"]', { + state: 'visible', + timeout: 30000, + }) + + // Inject mock application state so the UI review has something to analyze + await page.waitForFunction( + () => + !!(window as unknown as { __TEST_CONTROLS__: boolean }) + .__TEST_CONTROLS__ + ) + + // Build type-safe messages outside evaluate to ensure types are correct at compile time + const timerMessage: ServerMessage = { + type: 'TIMER_UPDATE', + payload: { + isRunning: true, + currentPhase: 'WORK', + timeRemaining: 15, + timeElapsed: 45, + caloriesBurned: 12, + mode: 'TABATA', + workDuration: 30, + restDuration: 10, + soundEventId: 0, + }, + } + + const hrmMessage: ServerMessage = { + type: 'HRM_UPDATE', + payload: [ + { + clientId: 'mock-1', + value: 155, + maxHr: 185, + zone: 'ZONE_3', + percentage: 85, + name: 'Mock Device', + calories: 120, + }, + ], + } + + const spotifyInitMessage: ServerMessage = { + type: 'SPOTIFY_SERVICE_INIT_UPDATE', + payload: true, + } + + const spotifyMessage: ServerMessage = { + type: 'SPOTIFY_UPDATE', + payload: { + devices: [], + playback: { + track: { + id: 'track-1', + name: 'UI Review Track', + artist: 'Gemini', + albumName: 'Review Album', + albumArtUrl: '', + }, + is_playing: true, + volume_percent: 50, + isMuted: false, + progress_ms: 30000, + }, + }, + } + + await page.evaluate( + ({ timerMsg, hrmMsg, spotifyInitMsg, spotifyMsg }) => { + const dispatch = ( + window as unknown as { + __TEST_CONTROLS__: { dispatch: (msg: ServerMessage) => void } + } + ).__TEST_CONTROLS__.dispatch + dispatch(timerMsg) + dispatch(hrmMsg) + dispatch(spotifyInitMsg) + dispatch(spotifyMsg) + }, + { + timerMsg: timerMessage, + hrmMsg: hrmMessage, + spotifyInitMsg: spotifyInitMessage, + spotifyMsg: spotifyMessage, + } + ) + + // Force layout stabilization for the screenshot + await page.addStyleTag({ + content: `[data-testid="main-content-layout"] { opacity: 1 !important; transform: none !important; }`, + }) + await page.waitForTimeout(2000) + + const screenshotBuffer = await page.screenshot({ fullPage: true }) + + const model = genAI.getGenerativeModel({ + model: 'gemini-1.5-pro', + }) + + const prompt = ` + Act as a Senior UX Architect and Accessibility Expert. + Analyze this HRM (Heart Rate Monitor) application screenshot for: + + 1. **Visual Regressions**: Identify overlapping text, broken Material-UI components, or alignment shifts. + 2. **WCAG Compliance**: Identify low-contrast areas (especially the Timer text against PhaseBackground) or missing focus indicators. + 3. **Hierarchy & UX**: Evaluate the primary CTA visibility. Is it clear to the user what the current state is (e.g., has the workout started)? + 4. **Material-UI Best Practices**: Check for consistent 16px gutters between cards and proper use of high-density list items. + 5. **Feedback**: Provide a concise list of actionable UI improvements. + + Be specific. Mention component names or areas of the screen. + Return your analysis as a markdown string. + ` + + const result = await model.generateContent([ + prompt, + { + inlineData: { + data: screenshotBuffer.toString('base64'), + mimeType: 'image/png', + }, + }, + ]) + + const response = await result.response + const feedback = + response.candidates?.[0]?.content?.parts?.[0]?.text || + 'No feedback generated.' + + const prNumber = process.env.PR_NUMBER + const githubToken = process.env.GITHUB_TOKEN + const repoFullName = process.env.GITHUB_REPOSITORY + + if (prNumber && githubToken && repoFullName) { + const body = `### 🤖 Gemini UI Review\n\n${feedback}\n\n---\n*This review was triggered by the @gemini-ui-review command.*` + const [owner, repo] = (repoFullName as string).split('/') + + const octokit = new Octokit({ auth: githubToken }) + await octokit.rest.issues.createComment({ + owner: owner as string, + repo: repo as string, + issue_number: parseInt(prNumber as string, 10), + body, + }) + console.log(`✅ Successfully posted UI review to PR #${prNumber}`) + } else { + console.log(feedback) + } + } catch (error) { + console.error('❌ UI review execution failed:', error) + process.exit(1) + } finally { + await browser.close().catch(console.error) + } +} + +performUIReview().catch((err) => { + console.error('💥 Fatal UI Review error:', err) + process.exit(1) +})