scan-issue-queue.yml

# This workflow scans ALL open issues with titles starting with "SCAN:"
# It runs daily at midnight UTC and can also be triggered manually.
# Use this workflow to process regular scan requests submitted via the web form.

name: Scan All Open SCAN Issues

on:
  schedule:
    # Run daily at midnight UTC
    - cron: '0 0 * * *'
  workflow_dispatch:
    # Allow manual triggering

permissions:
  contents: write
  issues: write
  models: read

concurrency:
  group: scan-repository
  cancel-in-progress: false

jobs:
  process-scan-queue:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Setup Node
        uses: actions/setup-node@v6
        with:
          node-version: 24

      - name: Install dependencies
        run: npm ci

      - name: Install Playwright browsers
        run: npx playwright install --with-deps chromium

      # Timeout configuration (optional, defaults shown):
      # - FETCH_TIMEOUT_MS: Maximum time for initial HTTP fetch (default: 30000ms = 30s)
      # - PER_URL_TIMEOUT_MS: Maximum time per URL including fetch and all audits (default: 60000ms = 60s)
      # - TOTAL_SCAN_TIMEOUT_MS: Maximum total scan time (default: 3000000ms = 50min)
      # - ALFA_PAGE_TIMEOUT_MS: ALFA CLI page load timeout (default: 30000ms = 30s)
      # - ALFA_COMMAND_TIMEOUT_MS: ALFA command execution timeout (default: 180000ms = 3min)
      # - PLAYWRIGHT_NAV_TIMEOUT_MS: Playwright page navigation timeout (default: 30000ms = 30s)
      # - PLAYWRIGHT_LAUNCH_TIMEOUT_MS: Playwright browser launch timeout (default: 30000ms = 30s)
      #
      # To customize timeouts, add environment variables to the spawnSync call in the "Scan each issue" step

      - name: Fetch open SCAN issues
        id: fetch-issues
        uses: actions/github-script@v8
        with:
          script: |
            const issues = await github.paginate(github.rest.issues.listForRepo, {
              owner: context.repo.owner,
              repo: context.repo.repo,
              state: 'open',
              per_page: 100
            });

            const scanIssues = issues.filter(issue =>
              issue.title.startsWith('SCAN:') && !issue.pull_request
            );

            console.log(`Found ${scanIssues.length} open SCAN issues`);
            return scanIssues.map(issue => issue.number);
          result-encoding: json

      - name: Scan each issue
        if: steps.fetch-issues.outputs.result != '[]'
        uses: actions/github-script@v8
        with:
          script: |
            const { execSync, spawnSync, spawn } = require('child_process');
            const fs = require('fs');
            const path = require('path');

            const issueNumbers = ${{ steps.fetch-issues.outputs.result }};

            if (!issueNumbers || issueNumbers.length === 0) {
              console.log('No SCAN issues to process');
              return;
            }

            console.log(`Processing ${issueNumbers.length} issues: ${issueNumbers.join(', ')}`);

            const defaultBranch = context.payload.repository.default_branch || 'main';
            console.log(`Using default branch: ${defaultBranch}`);

            for (const issueNumber of issueNumbers) {
              console.log(`\n=== Processing issue #${issueNumber} ===`);

              // Validate issue number is a positive integer
              if (!Number.isInteger(issueNumber) || issueNumber <= 0) {
                console.error(`Invalid issue number: ${issueNumber}`);
                continue;
              }

              const eventPath = `.tmp/issue-${issueNumber}-event.json`;
              const scanOutputDir = `.scan-output-${issueNumber}`;

              try {
                // Fetch the issue details
                const { data: issue } = await github.rest.issues.get({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  issue_number: issueNumber
                });

                // Create event payload for the scanner
                const eventPayload = {
                  action: 'scheduled-scan',
                  issue: issue
                };

                // Create temp directory and event file
                fs.mkdirSync('.tmp', { recursive: true });
                fs.writeFileSync(eventPath, JSON.stringify(eventPayload, null, 2));

                // Create scan output directory
                fs.mkdirSync(scanOutputDir, { recursive: true });

                // Run the scan using spawn for better security
                console.log(`Running scan for issue #${issueNumber}...`);
                const scanProcess = await new Promise((resolve, reject) => {
                  const child = spawn(
                    process.execPath,
                    ['scanner/run-scan.mjs', eventPath, scanOutputDir],
                    { env: process.env }
                  );

                  let stdout = '';
                  let stderr = '';

                  child.stdout.on('data', (chunk) => {
                    stdout += chunk.toString();
                  });

                  child.stderr.on('data', (chunk) => {
                    const text = chunk.toString();
                    stderr += text;
                    for (const line of text.split('\n')) {
                      const trimmed = line.trim();
                      if (trimmed) {
                        console.log(`[issue #${issueNumber}] ${trimmed}`);
                      }
                    }
                  });

                  child.on('error', (error) => {
                    reject(new Error(`Failed to run scan: ${error.message}`));
                  });

                  child.on('close', (code) => {
                    if (code !== 0) {
                      reject(new Error(`Scan process exited with code ${code}\n${stderr.slice(-5000)}`));
                      return;
                    }
                    resolve({ stdout, stderr });
                  });
                });

                const scanResult = scanProcess.stdout;
                let meta;
                try {
                  meta = JSON.parse(scanResult);
                } catch (parseError) {
                  throw new Error(`Failed to parse scan output as JSON: ${parseError.message}`);
                }
                console.log(`Scan completed:`, meta);

                const lastScannedMatch = [...scanProcess.stderr.matchAll(/\[(\d+)\/(\d+)\]\s+Scanned\s+(.+?)\s+in\s+(\d+)ms/g)].pop();
                if (lastScannedMatch) {
                  const [, current, total, url, elapsedMs] = lastScannedMatch;
                  console.log(`Last scanned URL for issue #${issueNumber}: ${url} (${elapsedMs}ms, ${current}/${total})`);
                }

                if (meta.skipped) {
                  console.log(`Issue #${issueNumber} was skipped: ${meta.reason}`);
                  // Clean up temporary files
                  fs.rmSync(eventPath, { force: true });
                  fs.rmSync(scanOutputDir, { recursive: true, force: true });
                  continue;
                }

                // If crawl mode was used, update the issue body with discovered URLs
                // so future re-runs use the same URL list instead of crawling again.
                if (meta.discoveredUrls && meta.discoveredUrls.length > 0) {
                  try {
                    const { data: issueData } = await github.rest.issues.get({
                      owner: context.repo.owner,
                      repo: context.repo.repo,
                      issue_number: issueNumber
                    });
                    const urlsList = meta.discoveredUrls.join('\n');
                    let updatedBody = (issueData.body || '').replace(/^Page:\s*\d+\s*$/m, urlsList);
                    await github.rest.issues.update({
                      owner: context.repo.owner,
                      repo: context.repo.repo,
                      issue_number: issueNumber,
                      body: updatedBody
                    });
                    console.log(`Updated issue #${issueNumber} body with ${meta.discoveredUrls.length} discovered URLs`);
                  } catch (updateError) {
                    console.warn(`Failed to update issue #${issueNumber} with discovered URLs:`, updateError.message);
                  }
                }

                // Prepare report paths
                const stamp = new Date().toISOString().replace(/[:.]/g, '-');
                const reportDir = `reports/issues/issue-${issueNumber}/${stamp}`;
                const summaryPath = `${reportDir}/report.json`;
                const markdownPath = `${reportDir}/report.md`;
                const htmlPath = `${reportDir}/report.html`;
                const csvPath = `${reportDir}/report.csv`;

                // Create report directory and copy files
                fs.mkdirSync(reportDir, { recursive: true });
                fs.copyFileSync(`${scanOutputDir}/report.json`, summaryPath);
                fs.copyFileSync(`${scanOutputDir}/report.md`, markdownPath);
                fs.copyFileSync(`${scanOutputDir}/report.html`, htmlPath);
                fs.copyFileSync(`${scanOutputDir}/report.csv`, csvPath);

                // Clean up temporary files
                fs.rmSync(eventPath, { force: true });
                fs.rmSync(scanOutputDir, { recursive: true, force: true });

                // Regenerate reports.html
                console.log('Regenerating reports.html...');
                const generateResult = spawnSync(process.execPath, ['scanner/generate-reports-html.mjs']);
                if (generateResult.status !== 0) {
                  console.warn('Failed to regenerate reports.html:', generateResult.stderr);
                }

                // Commit and push the report using spawnSync for security
                spawnSync('git', ['config', 'user.name', 'github-actions[bot]']);
                spawnSync('git', ['config', 'user.email', '41898282+github-actions[bot]@users.noreply.github.com']);
                spawnSync('git', ['add', reportDir]);
                spawnSync('git', ['add', 'reports.html', 'trends.html', 'stats.json']);

                // Check if there are changes to commit
                const statusResult = spawnSync('git', ['status', '--porcelain'], { encoding: 'utf8' });
                if (statusResult.stdout.trim()) {
                  const commitMessage = `chore(scan): scheduled scan for issue #${issueNumber}`;
                  const commitResult = spawnSync('git', ['commit', '-m', commitMessage]);
                  if (commitResult.status !== 0) {
                    throw new Error(`Failed to commit: ${commitResult.stderr}`);
                  }
                  
                  // Retry push with rebase up to 3 times
                  let pushSucceeded = false;
                  for (let attempt = 1; attempt <= 3; attempt++) {
                    console.log(`Push attempt ${attempt}...`);
                    const pullResult = spawnSync('git', ['pull', '--rebase', 'origin', defaultBranch], { encoding: 'utf8' });
                    if (pullResult.status === 0) {
                      const pushResult = spawnSync('git', ['push', 'origin', `HEAD:${defaultBranch}`], { encoding: 'utf8' });
                      if (pushResult.status === 0) {
                        console.log(`Report committed and pushed for issue #${issueNumber} on attempt ${attempt}`);
                        pushSucceeded = true;
                        break;
                      } else {
                        console.warn(`Push failed on attempt ${attempt}: ${pushResult.stderr}`);
                      }
                    } else {
                      console.warn(`Rebase failed on attempt ${attempt}: ${pullResult.stderr}`);
                      // reports.html is a generated file — resolve conflicts by regenerating it
                      const statusCheck = spawnSync('git', ['status'], { encoding: 'utf8' });
                      if (statusCheck.stdout.includes('rebase in progress')) {
                        const conflictCheck = spawnSync('git', ['diff', '--name-only', '--diff-filter=U'], { encoding: 'utf8' });
                        if (conflictCheck.stdout.trim().split('\n').includes('reports.html')) {
                          console.log('Conflict in generated reports.html — resolving by regenerating...');
                          spawnSync(process.execPath, ['scanner/generate-reports-html.mjs']);
                          spawnSync('git', ['add', 'reports.html', 'trends.html', 'stats.json']);
                          const continueResult = spawnSync('git', ['rebase', '--continue'], { encoding: 'utf8', env: { ...process.env, GIT_EDITOR: 'true' } });
                          if (continueResult.status === 0) {
                            const pushResult = spawnSync('git', ['push', 'origin', `HEAD:${defaultBranch}`], { encoding: 'utf8' });
                            if (pushResult.status === 0) {
                              console.log(`Report committed and pushed for issue #${issueNumber} on attempt ${attempt} (after resolving reports.html conflict)`);
                              pushSucceeded = true;
                              break;
                            }
                          }
                        }
                        spawnSync('git', ['rebase', '--abort']);
                      }
                    }
                    
                    if (attempt < 3) {
                      const delay = attempt * 2000; // Wait 2s before 2nd attempt, 4s before 3rd attempt
                      console.log(`Waiting ${delay}ms before retry...`);
                      await new Promise(resolve => setTimeout(resolve, delay));
                    }
                  }
                  
                  if (!pushSucceeded) {
                    throw new Error(`Failed to push after 3 attempts`);
                  }
                } else {
                  console.log(`No changes to commit for issue #${issueNumber}`);
                }

                // Post comment with results
                const owner = context.repo.owner;
                const repo = context.repo.repo;
                const encodedMarkdownPath = markdownPath.split('/').map(encodeURIComponent).join('/');
                const encodedHtmlPath = htmlPath.split('/').map(encodeURIComponent).join('/');
                const encodedCsvPath = csvPath.split('/').map(encodeURIComponent).join('/');
                const pagesUrl = `https://${owner}.github.io/${repo}/${encodedHtmlPath}`;
                const csvPagesUrl = `https://${owner}.github.io/${repo}/${encodedCsvPath}`;
                const blobUrl = `https://github.com/${owner}/${repo}/blob/${encodeURIComponent(defaultBranch)}/${encodedMarkdownPath}`;
                const csvBlobUrl = `https://github.com/${owner}/${repo}/blob/${encodeURIComponent(defaultBranch)}/${encodedCsvPath}`;

                const body = [
                  `Scheduled scan complete for **${meta.scanTitle}**.`,
                  "",
                  `- Accepted URLs: ${meta.acceptedCount}`,
                  `- Scanned URLs: ${meta.scannedCount ?? meta.acceptedCount}`
                ];
                
                // Add warning if some URLs were skipped
                if (meta.skippedDueToTimeout && meta.skippedDueToTimeout > 0) {
                  body.push(`- ⚠️ **${meta.skippedDueToTimeout} URLs skipped due to timeout** (scan exceeded time limit)`);
                }
                
                // Add recommendation if only 1 URL was scanned
                if (meta.acceptedCount === 1) {
                  body.push(``);
                  body.push(`💡 **Only 1 URL was scanned.** Open Scans works best with multiple URLs. To generate a list of related URLs for a more thorough accessibility scan, try the [Top Task Finder](https://mgifford.github.io/top-task-finder/). You can edit this issue to add more URLs and reopen it to trigger a new scan.`);
                  body.push(``);
                }
                
                body.push(
                  `- Rejected URLs: ${meta.rejectedCount}`,
                  "",
                  `Report (Pages): ${pagesUrl}`,
                  `Fallback (GitHub): ${blobUrl}`,
                  `CSV (Pages): ${csvPagesUrl}`,
                  `CSV Fallback (GitHub): ${csvBlobUrl}`
                );
                
                const commentBody = body.join("\n");

                // Retry posting comment with exponential backoff
                let commentPosted = false;
                for (let attempt = 0; attempt < 3; attempt++) {
                  try {
                    await github.rest.issues.createComment({
                      owner,
                      repo,
                      issue_number: issueNumber,
                      body: commentBody
                    });
                    console.log(`Comment posted to issue #${issueNumber}`);
                    commentPosted = true;
                    break;
                  } catch (commentError) {
                    console.warn(`Attempt ${attempt + 1} failed to post comment:`, commentError.message);
                    if (attempt < 2) {
                      const delay = Math.pow(2, attempt) * 1000; // 1s, 2s
                      console.log(`Retrying in ${delay}ms...`);
                      await new Promise(resolve => setTimeout(resolve, delay));
                    }
                  }
                }
                
                if (!commentPosted) {
                  console.error(`Failed to post success comment after 3 attempts`);
                }

                // Close the issue if URLs were successfully scanned
                // Note: Issues with zero accepted URLs remain open for review
                if (meta.acceptedCount > 0) {
                  let issueClosed = false;
                  for (let attempt = 0; attempt < 3; attempt++) {
                    try {
                      await github.rest.issues.update({
                        owner,
                        repo,
                        issue_number: issueNumber,
                        state: 'closed'
                      });
                      console.log(`Issue #${issueNumber} closed`);
                      issueClosed = true;
                      break;
                    } catch (closeError) {
                      console.warn(`Attempt ${attempt + 1} failed to close issue:`, closeError.message);
                      if (attempt < 2) {
                        const delay = Math.pow(2, attempt) * 1000; // 1s, 2s
                        console.log(`Retrying in ${delay}ms...`);
                        await new Promise(resolve => setTimeout(resolve, delay));
                      }
                    }
                  }
                  
                  if (!issueClosed) {
                    console.error(`Failed to close issue after 3 attempts. Scan completed successfully but issue remains open.`);
                  }
                }

              } catch (error) {
                console.error(`Error processing issue #${issueNumber}:`, error);

                // Clean up temporary files on error
                try {
                  fs.rmSync(eventPath, { force: true });
                  fs.rmSync(scanOutputDir, { recursive: true, force: true });
                } catch (cleanupError) {
                  console.error(`Failed to clean up temporary files:`, cleanupError);
                }

                // Post error comment (no retry - best effort)
                try {
                  await github.rest.issues.createComment({
                    owner: context.repo.owner,
                    repo: context.repo.repo,
                    issue_number: issueNumber,
                    body: `❌ Scheduled scan encountered an error: ${error.message}\n\nPlease check the [workflow logs](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) for details.`
                  });
                } catch (commentError) {
                  console.error(`Failed to post error comment:`, commentError.message);
                }
              }
            }