refactor(flaky-history-analysis): enhance state management and analysis windows

- Updated the state marker in `flaky-history-analysis.ts` and `flaky-sticky-comment.ts` to use base64 encoding, ensuring safe handling of payloads containing HTML comment sequences.
- Adjusted the analysis window configuration in `flaky-history-analysis.ts` and the corresponding workflow file to include a 15-day bucket alongside the existing 7-day and 30-day windows.
- Modified the `.gitignore` to include a new generated artifact from the flaky unit test detection workflow.
- Added a test comment in `index.test.tsx` to trigger the flaky test detection process.

Author: NicolasMassart

.github/scripts/flaky-history-analysis.ts

@@ -39,8 +39,11 @@ import { dirname, join } from 'path';
 type Octokit = ReturnType<typeof getOctokit>;
 
 // Prefix of the hidden state block embedded in the sticky comment body.
-// Stage 1 reads this to determine which files need re-analysis.
-const STATE_MARKER = '<!-- metamask-flaky-test-detection:state';
+// Stage 1 reads this to determine which files need re-analysis. The state
+// payload is base64-encoded (see flaky-sticky-comment.ts buildStateBlock), so
+// the marker name says so and the base64 alphabet can never contain the `-->`
+// sequence that closes the HTML comment.
+const STATE_MARKER = '<!-- metamask-flaky-test-detection-metadata=';
 // Marker used by Stage 3 to identify the sticky comment (must stay in sync).
 const COMMENT_MARKER = '<!-- metamask-flaky-test-detection -->';
 
@@ -58,7 +61,7 @@ const UNIT_TEST_JOB_PREFIX = 'Unit tests';
 // nested windows below, so a failure 5 days ago counts in both windows and
 // one 20 days ago counts only in the 30d bucket.
 const LOOKBACK_DAYS = 30;
-const WINDOWS_DAYS = [7, 30] as const;
+const WINDOWS_DAYS = [7, 15, 30] as const;
 type WindowKey = `${(typeof WINDOWS_DAYS)[number]}d`;
 const WINDOW_KEYS = WINDOWS_DAYS.map((d) => `${d}d` as WindowKey);
 type WindowCounts = Record<WindowKey, number>;
@@ -193,14 +196,19 @@ async function mapWithConcurrency<T, R>(
 }
 
 // Extracts the per-file state JSON block embedded in a sticky comment body.
+// The payload is base64-encoded, so decoding it is safe even if the AI
+// findings inside contain a literal `-->` — that sequence cannot occur inside
+// base64 output, only in the (correctly detected) closing delimiter.
 function parseStateFromComment(body: string): CommentState | null {
   const idx = body.indexOf(STATE_MARKER);
   if (idx === -1) return null;
   const after = body.slice(idx + STATE_MARKER.length).trimStart();
   const closeIdx = after.indexOf(' -->');
   if (closeIdx === -1) return null;
+  const encoded = after.slice(0, closeIdx).trim();
   try {
-    return JSON.parse(after.slice(0, closeIdx)) as CommentState;
+    const json = Buffer.from(encoded, 'base64').toString('utf8');
+    return JSON.parse(json) as CommentState;
   } catch {
     return null;
   }
@@ -582,4 +590,8 @@ async function main(): Promise<void> {
 
 main().catch((error: Error) => {
   core.warning(`Stage 1 failed: ${error.message}`);
+  // A crash after should_analyze was emitted must not let Stage 3 run against a
+  // missing history artifact and post a false all-clear.
+  core.setOutput('should_analyze', 'false');
+  core.setOutput('files_to_analyze', '');
 });

.github/scripts/flaky-sticky-comment.ts

@@ -27,7 +27,9 @@ import { join } from 'path';
 const MARKER = '<!-- metamask-flaky-test-detection -->';
 // Prefix of the hidden state block appended to the comment body. Stage 1 reads
 // this on the next push to determine which files changed since last analysis.
-const STATE_MARKER = '<!-- metamask-flaky-test-detection:state';
+// The state payload is base64-encoded (see buildStateBlock below),
+// so base64 alphabet can never contain the `-->` sequence that closes the HTML comment.
+const STATE_MARKER = '<!-- metamask-flaky-test-detection-metadata=';
 // Canonical source of the flaky-test-detection skill. The synced copy at
 // .agents/skills/mms-flaky-test-detection/SKILL.md is .gitignore'd, so we link
 // to the real source repo (MetaMask/skills) instead of a 404 blob path.
@@ -128,9 +130,12 @@ const DEFAULT_WINDOWS = [7, 15, 30];
 
 // Serializes per-file state into a hidden HTML comment appended to the comment
 // body. Stage 1 reads this on the next push to decide which files need
-// re-analysis vs. which can be preserved verbatim.
+// re-analysis vs. which can be preserved verbatim. The JSON is base64-encoded
+// so an AI finding's snippet/explanation/suggestedFix containing a literal
+// `-->` cannot truncate the payload before the real closing delimiter.
 function buildStateBlock(state: CommentState): string {
-  return `${STATE_MARKER} ${JSON.stringify(state)} -->`;
+  const encoded = Buffer.from(JSON.stringify(state), 'utf8').toString('base64');
+  return `${STATE_MARKER}${encoded} -->`;
 }
 
 // Builds the table (or an empty-state fallback line) for the "Run history
@@ -301,6 +306,11 @@ async function main(): Promise<void> {
     return;
   }
 
+  if (!existsSync(HISTORY_PATH)) {
+    console.log('⏭️  History artifact missing — Stage 1 did not complete, skipping');
+    return;
+  }
+
   const [owner, repo] = env.repo.split('/');
   const octokit = getOctokit(env.token);
 

.github/workflows/flaky-unit-test-detection.yml

@@ -5,7 +5,7 @@
 #
 #   1. Deterministic history — flaky-history-analysis.ts counts how often each
 #      modified test file failed on main over the last 30 days of completed
-#      ci.yml runs, bucketed into 7d/30d windows.
+#      ci.yml runs, bucketed into 7d/15d/30d windows.
 #      → writes .ai-pr-analyzer/flaky-history.json
 #
 #   2. AI analyzer — MetaMask/ai-analyzer's flaky-unit-test-analysis mode
@@ -69,8 +69,8 @@ jobs:
           command: cd .github/scripts && yarn --immutable
 
       # Stage 1: deterministic historical analysis. Identifies modified unit
-      # test files and counts their failures on main over the last 365 days of
-      # completed ci.yml runs, bucketed into 7d/30d/90d/365d windows
+      # test files and counts their failures on main over the last 30 days of
+      # completed ci.yml runs, bucketed into 7d/15d/30d windows
       # (failure+success runs only; cancelled/timed_out are excluded from the
       # denominator). When no modified test file changed since its last-analyzed
       # SHA (stored in the sticky comment), the step emits should_analyze=false
@@ -95,12 +95,19 @@ jobs:
       # The synced SKILL.md is then copied into .ai-pr-analyzer/skills/ so the
       # analyzer's load_skill tool can find it (single source of truth — no
       # hand-duplicated pattern content in this repo).
+      # Pinned to a reviewed commit SHA rather than `main`: this bootstrap runs
+      # in the same workspace as later token-bearing steps, so executing code
+      # from a moving branch would let an unreviewed upstream change affect
+      # this PR's CI run. Bump the SHA deliberately when the skill needs
+      # updating.
       - name: Sync flaky-test-detection skill
         if: >-
           steps.detect-history.outputs.should_analyze == 'true' &&
           !github.event.pull_request.head.repo.fork
+        env:
+          SKILLS_BOOTSTRAP_SHA: fad21fb8f6158a8f939e8b30bf8f8af755ce67cd
         run: |
-          curl -fsSL https://raw.githubusercontent.com/MetaMask/skills/main/tools/bootstrap \
+          curl -fsSL "https://raw.githubusercontent.com/MetaMask/skills/${SKILLS_BOOTSTRAP_SHA}/tools/bootstrap" \
             | bash -s -- --repo metamask-mobile --domain coding
           mkdir -p .ai-pr-analyzer/skills
           cp .agents/skills/mms-flaky-test-detection/SKILL.md \
@@ -134,6 +141,10 @@ jobs:
       # test files — the composite action always analyzes the full PR diff.
       # This also means the analyzer never has a chance to post its own PR
       # comment; Stage 3 below is the only PR-visible output.
+      # --skip-scope bypasses the analyzer's default minimum-changed-lines
+      # gate: Stage 1 already narrowed the input to modified unit test files,
+      # so a 1-2 line edit that introduces a flaky pattern must still be
+      # analyzed instead of being skipped for being "too small".
       - name: Run flaky unit test AI analysis
         if: >-
           steps.detect-history.outputs.should_analyze == 'true' &&
@@ -149,14 +160,20 @@ jobs:
             --config .ai-pr-analyzer \
             --mode flaky-unit-test-analysis \
             --base-branch "origin/${{ github.base_ref }}" \
-            --changed-files "${{ steps.detect-history.outputs.files_to_analyze }}"
+            --changed-files "${{ steps.detect-history.outputs.files_to_analyze }}" \
+            --skip-scope
 
       # Stage 3: unified sticky PR comment combining Stage 1 + Stage 2
-      # output. Runs only when should_analyze is true — if Stage 1 determined
-      # no modified test file changed since last analysis, the existing comment
-      # is left untouched and Stage 3 is skipped entirely. Never blocks the PR.
+      # output. Runs when should_analyze is true, or when the PR no longer
+      # modifies any unit test file — the latter lets the 4-state matrix flip
+      # a stale sticky comment to "all fixed" when test changes are removed.
+      # If Stage 1 determined no modified test file changed since last
+      # analysis (should_analyze=false but has_test_files=true), the existing
+      # comment is left untouched and Stage 3 is skipped. Never blocks the PR.
       - name: Post unified sticky comment
-        if: steps.detect-history.outputs.should_analyze == 'true'
+        if: >-
+          steps.detect-history.outputs.should_analyze == 'true' ||
+          steps.detect-history.outputs.has_test_files == 'false'
         continue-on-error: true
         working-directory: .github/scripts
         env:

.gitignore

@@ -221,12 +221,8 @@ release-signoffs.json
 .claude/skills/
 .claude/commands/
 .agents/skills/
-# Skills copied at CI time by .github/workflows/flaky-unit-test-detection.yml
-# from .agents/skills/mms-*/SKILL.md into ai-analyzer's skill lookup path.
 .ai-pr-analyzer/skills/*
-# Generated artifacts from the flaky-unit-test-detection workflow (Stage 1).
-.ai-pr-analyzer/flaky-history.json
-.ai-pr-analyzer/flaky-prior-state.json
+.ai-pr-analyzer/flaky*
 .cursor/rules/*
 !.cursor/rules/*.mdc
 .cursor/skills/

app/components/Base/RemoteImage/index.test.tsx

@@ -1,3 +1,4 @@
+// [mcwp-474-tmp]
 import React from 'react';
 import RemoteImage from './';
 import { getFormattedIpfsUrl } from '@metamask/assets-controllers';