Merge branch 'main' into fix/cspm-gcp-flaky-tests

Author: maxcold

.agents/skills/flaky-test-investigator/SKILL.md

@@ -42,8 +42,6 @@ For every failure, try to retrieve:
 - **Server logs** (`kibana.log`, `elasticsearch.log` when present). Cross-reference the failure timestamp with any errors in the logs — a server-side 500 or unexpected warning is strong evidence the failure is a product bug, not a test bug.
 - **Full session trace** when the framework supports it (Scout / Playwright). Lets you scrub through every step, locator query, network call, and DOM snapshot.
 
-How to actually find and download each artifact type is framework-specific — see "Retrieve failure artifacts" below.
-
 Things to specifically check in the artifacts before forming a root-cause hypothesis:
 
 - **Did the expected element render at all?** If yes and the selector missed it → flaky selector (Tier 2 fix territory). If no → real rendering / race / data issue (Tier 1 territory).
@@ -53,27 +51,9 @@ Things to specifically check in the artifacts before forming a root-cause hypoth
 
 If artifacts are not available (expired, not uploaded, no `read_artifacts` token), say so in the report rather than fabricating a hypothesis. "Screenshot would have resolved this; not available" is a valid open question.
 
-### Retrieve failure artifacts
-
-The standard recipe is **list → filter by path → download by ID**, always scoped to the failed job's UUID. Two Buildkite gotchas to know about first:
-
-- **Failed-attempt jobs are hidden by default.** `/builds/<n>` returns only the latest attempt; append `?include_retried_jobs=true` to find the original failing job (the one cited in `failed-test` comments). `retried` and `retried_in_job_id` link the two.
-- **Per-job artifacts use a different endpoint than build-wide artifacts.** If a build retried to green, failure artifacts only live on the failed job's listing (`bk artifacts list <build> -p <pipeline> --job-uuid <jobId>`). Don't conclude "no screenshot uploaded" until you've checked there.
-
-**Scout** (`@kbn/scout-reporting`, not standard Playwright output — `playwright-report/`, `trace.zip`, and video are NOT published):
-
-- `.scout/reports/scout-playwright-test-failures-<runId>/test-failures-summary.json` — maps test name → HTML report. Start here.
-- `.scout/reports/scout-playwright-test-failures-<runId>/<testId>.html` — self-contained: error, stdout, embedded screenshot. Usually sufficient on its own.
-- `.scout/reports/scout-playwright-test-failures-<runId>/scout-failures-<runId>.ndjson` — one record per failure (`id` = `<testId>`, `owner`, `location`, `error.*`) for programmatic use.
-- `**/.scout/test-artifacts/<test-slug>/test-failed-<N>.png` — plain Playwright screenshot; the PNG doesn't carry `<testId>`, so correlate via spec path.
-
-**FTR** (a single content `<hash>` links every artifact for one failure):
-
-- `target/test_failures/<jobId>_<hash>.{json,log,html}` — `.json` is source of truth; full Kibana/ES stdout lives in `system-out` (there is no separate `kibana.log`). Pull this first.
-- `<test-root>/screenshots/failure/*-<hash>.png` and `<test-root>/failure_debug/html/*-<hash>.html` — UI tests only; fetch only when the failure is UI-side.
-- `.es/*.log` — transport/cluster-shaped failures.
+### List failure artifacts
 
-`target/test_failures/` is shared with Scout; filter by `.jobName` (e.g. `FTR Configs #90` vs `Scout Lane #12`) to keep only FTR. On Cloud FTR pipelines the layout differs: one self-contained HTML per failure at `<config-path-with-underscores>-<unix-timestamp>/html/<contentHash>.html` — no `target/test_failures/`, screenshot, or DOM artifacts.
+`bk artifacts list <build> -p <pipeline> --job-uuid <jobId> --json` returns a JSON listing of every artifact uploaded for the failing job. Pass `--job-uuid <jobId>` for the failed attempt (without it, `bk` only returns the latest attempt and hides retried failures). If a build retried to green, failure artifacts only live on the failed job's listing; don't conclude "no screenshot" until you've scoped to the right job UUID.
 
 ### Understand the scope
 

.github/CODEOWNERS

@@ -350,6 +350,7 @@ src/core/packages/user-profile/server-mocks @elastic/kibana-core
 src/core/packages/user-settings/server @elastic/kibana-security
 src/core/packages/user-settings/server-internal @elastic/kibana-security
 src/core/packages/user-settings/server-mocks @elastic/kibana-security
+src/core/packages/user-settings/types @elastic/kibana-security
 src/core/packages/user-storage/browser @elastic/appex-sharedux
 src/core/packages/user-storage/browser-internal @elastic/appex-sharedux
 src/core/packages/user-storage/browser-mocks @elastic/appex-sharedux
@@ -3543,6 +3544,7 @@ x-pack/solutions/observability/plugins/synthetics/server/saved_objects/synthetic
 /src/cli/ @elastic/kibana-operations
 /src/cli_keystore/ @elastic/kibana-operations
 /.github/workflows/ @elastic/kibana-operations
+/.github/workflows/failed-test-investigator.md @elastic/kibana-operations @elastic/appex-qa
 /.github/aw/ @elastic/kibana-operations
 /.buildkite/ @elastic/kibana-operations
 /moon.yml @elastic/kibana-operations

.github/workflows/failed-test-investigator.lock.yml

@@ -27,6 +27,8 @@ concurrency:
 
 env:
   ISSUE_NUMBER: &issue_number ${{ github.event.issue.number || github.event.inputs.issue_number }}
+  # Lets the agent omit `-o elastic` on every `bk` invocation (see https://buildkite.com/docs/pipelines/configure/environment-variables)
+  BUILDKITE_ORGANIZATION_SLUG: elastic
 
 engine:
   id: claude
@@ -53,13 +55,36 @@ network:
     - defaults
     - buildkite.com
     - '*.buildkite.com'
+    - buildkiteartifacts.com
     - ci-stats.kibana.dev
     - github.com
     - api.github.com
     - chatgpt.com
     - elastic.litellm-prod.ai
 sandbox:
   agent: awf # Migrated from deprecated network setting
+steps:
+  - name: Install Buildkite CLI and export BUILDKITE_API_TOKEN
+    env:
+      BK_VERSION: 3.44.0
+      BK_SHA256: 88867c0b983ad2afe1efc26f0df6b46b5673577c1aea95eba76992636fb9abe9
+      OPS_BUILDKITE_TOKEN: ${{ secrets.OPS_BUILDKITE_TOKEN }}
+    run: |
+      set -euo pipefail
+      tmp="$(mktemp -d)"
+      url="https://github.com/buildkite/cli/releases/download/v${BK_VERSION}/bk_${BK_VERSION}_linux_amd64.tar.gz"
+      curl -fsSL --retry 3 --retry-delay 2 "${url}" -o "${tmp}/bk.tgz"
+      echo "${BK_SHA256}  ${tmp}/bk.tgz" | sha256sum -c -
+      tar -xzf "${tmp}/bk.tgz" -C "${tmp}" --strip-components=1 "bk_${BK_VERSION}_linux_amd64/bk"
+      install -d "${RUNNER_TEMP}/gh-aw/mcp-cli/bin"
+      install -m 0755 "${tmp}/bk" "${RUNNER_TEMP}/gh-aw/mcp-cli/bin/bk"
+      "${RUNNER_TEMP}/gh-aw/mcp-cli/bin/bk" --version
+      if [ -z "${OPS_BUILDKITE_TOKEN:-}" ]; then
+        echo "::error::OPS_BUILDKITE_TOKEN secret is not set" >&2
+        exit 1
+      fi
+      echo "BUILDKITE_API_TOKEN=${OPS_BUILDKITE_TOKEN}" >> "${GITHUB_ENV}"
+
 safe-outputs:
   noop:
     report-as-issue: false
@@ -89,7 +114,7 @@ Investigate a failed-test issue, classify the failure, and propose a fix when ap
 
 ## Investigate
 
-Investigate the test failure(s) using the `flaky-test-investigator` skill.
+Investigate the test failure(s) using the `flaky-test-investigator` skill. Use all of the data at your disposal to reach a conclusion (source code, logs, failure screenshots, etc.).
 
 Every conclusion must cite specific evidence. Do not guess.
 
@@ -139,32 +164,42 @@ No other side-effects beyond posting the comment and updating the label.
 
 ## Comment format
 
-Post exactly one comment. Keep the visible portion very short and easy to read:
+Post exactly one comment on the issue. Keep it concise, actionable, and prioritize the most critical findings at the very top. Adapt the sections below to best fit the specific failure. **Use `####` for all subsections** (e.g., `#### Proposed Fix`, `#### Root Cause`).
+
+Do not create standalone sections for "what the test does" "evidence," "where the test ran," or "failure screenshot". Integrate these details seamlessly into the sections below if they add value. Do not also mention why the `ai:auto-flaky-fix` isn't added.
+
+### 1. The TL;DR (Required)
+
+Start with a clear heading, essential metadata, and a brief summary of the failure, followed by a horizontal rule.
+
+```
+## {Classification}: {One-line description of what broke}
+
+## **Classification:** {type} | **Confidence:** {level} | **Introduced by:** {commit/PR if known}
+
+**Summary:** One or two sentences explaining the exact failure point.
+```
+
+### 2. Proposed fix (required)
 
-1. **One-line bold headline** stating the result kind and one identifying detail.
-2. **Diagnosis** (≤5 concise bullet points): what broke and where, the most likely root cause.
-3. **Next steps** (≤5 concise bullet points).
+Provide the most direct path to resolution immediately after the summary.
 
-Put the full `flaky-test-investigator` skill output inside a collapsed `<details><summary>Investigation details</summary> ... </details>` block (not in the visible portion). Open the block with a `#### Findings` subsection containing exactly these four bullets in this order — downstream tooling parses them, so preserve keys, casing, and `` - `key`: value `` shape. These bullets must live **inside `<details>`**, never in the visible portion:
+- **Single file:** lead directly with the suggested code diff or specific action.
+- **Multiple files:** use a brief table to list affected files, followed by the necessary changes.
+- **No concrete fix:** clearly state what additional evidence or investigation is needed to propose one.
 
-- `classification`: `test-design` | `test-environment` | `application` | `external` | `inconclusive`
-- `confidence`: `high` | `medium` | `low`
-- `test.type`: `scout` (if `scout-playwright` label) | `ftr` | `jest` | `unknown`
-- `test.file`: repo-relative path, or `unknown`
+### 3. Root Cause & Evidence (required)
 
-The skill's "Reporting" subsections should also be inside the collapsible section:
+Explain _why_ the failure occurred, citing specific evidence. Choose the format that best fits the complexity of the bug:
 
-- What the test does
-- What failed and when
-- Where it ran
-- Root cause hypothesis
-- Evidence
-- Failure screenshot
-- Recommended next step
-- Open questions
+- Use concise paragraphs with inline Markdown links pointing to specific code lines, commits, or files.
+- Use an ASCII timeline diagram for race conditions, multi-component bugs, or complex state leaks.
+- Fold relevant evidence (like missing `data-test-subj` attributes, failing network calls, or screenshot descriptions) directly into this narrative.
 
-Blank lines around `</summary>` and `</details>` are required for the inner markdown to render.
+### 4. Additional context (optional)
 
-End the comment with this footer line (verbatim, on its own line after the `</details>` block):
+Include the following only if they provide high-value, actionable signal:
 
-`<sup>AI-generated, share feedback in [#appex-qa](https://elastic.slack.com/archives/C04HT4P1YS3)</sup>`
+- **Ruled out:** a brief note on alternative hypotheses that were investigated and dismissed.
+- **Verification:** specific steps to reproduce the failure or confirm the fix.
+- **Open questions:** unresolved design or environmental issues blocking a definitive fix ("a screenshot would have helped troubleshoot this" is a valid open question).

.github/workflows/failed-test-investigator.md

@@ -537,6 +537,7 @@
     "@kbn/core-user-settings-server": "link:src/core/packages/user-settings/server",
     "@kbn/core-user-settings-server-internal": "link:src/core/packages/user-settings/server-internal",
     "@kbn/core-user-settings-server-mocks": "link:src/core/packages/user-settings/server-mocks",
+    "@kbn/core-user-settings-types": "link:src/core/packages/user-settings/types",
     "@kbn/core-user-storage-browser": "link:src/core/packages/user-storage/browser",
     "@kbn/core-user-storage-browser-internal": "link:src/core/packages/user-storage/browser-internal",
     "@kbn/core-user-storage-common": "link:src/core/packages/user-storage/common",

package.json

@@ -177,6 +177,7 @@ export const AppMenuActionButton = (props: AppMenuActionButtonProps) => {
         popoverWidth={popoverWidth}
         popoverTestId={popoverTestId}
         anchorPosition="downRight"
+        repositionToCrossAxis={false}
         onClose={onPopoverClose}
         onCloseOverflowButton={onCloseOverflowButton}
       />
@@ -195,6 +196,7 @@ export const AppMenuActionButton = (props: AppMenuActionButtonProps) => {
         popoverWidth={popoverWidth}
         popoverTestId={popoverTestId}
         anchorPosition="downRight"
+        repositionToCrossAxis={false}
         onClose={onPopoverClose}
         onCloseOverflowButton={onCloseOverflowButton}
       />

src/core/packages/chrome/app-menu/core-chrome-app-menu-components/src/components/app_menu_action_button.tsx

@@ -30,6 +30,7 @@ interface AppMenuContextMenuProps {
   switchConfig?: AppMenuSwitch;
   popoverTestId?: string;
   anchorPosition?: PopoverAnchorPosition;
+  repositionToCrossAxis?: boolean;
   onClose: () => void;
   onCloseOverflowButton?: () => void;
 }
@@ -47,6 +48,7 @@ export const AppMenuPopover = ({
   switchConfig,
   popoverTestId = 'app-menu-popover',
   anchorPosition = 'downLeft',
+  repositionToCrossAxis,
   onClose,
   onCloseOverflowButton,
 }: AppMenuContextMenuProps) => {
@@ -100,6 +102,7 @@ export const AppMenuPopover = ({
       hasArrow={false}
       anchorPosition={anchorPosition}
       aria-label={title || content}
+      repositionToCrossAxis={repositionToCrossAxis}
     >
       <EuiContextMenu initialPanelId={0} panels={panels} css={{ minWidth: 180 }} />
     </EuiPopover>

src/core/packages/chrome/app-menu/core-chrome-app-menu-components/src/components/app_menu_popover.tsx

@@ -60,9 +60,30 @@ export interface UserProfileUserInfo {
 }
 
 /**
- * Placeholder for data stored in user profile.
+ * Placeholder for data stored in user profile,
+ * services that store data in the user profile should specify said data by augmenting this type in it's implementation,
+ * like so:
+ *
+ * @example
+ * ```ts
+ * declare module '@kbn/core-user-profile-common' {
+ *   interface UserProfileData {
+ *     myService: {
+ *       myData: string;
+ *     };
+ *   }
+ * }
+ * ```
+ * This will make it such that the return value for the invocation of `getCurrent` is typed matching the defined augmentation
+ *
+ * ```ts
+ * const userProfile = await userProfileService.getCurrent();
+ * // accessing 'myService.myData' is now typed as 'string'
+ * console.log(userProfile.data.myService.myData);
+ * ```
  */
-export type UserProfileData = Record<string, unknown>;
+// eslint-disable-next-line @typescript-eslint/no-empty-interface -- See the comment above for an explanation.
+export interface UserProfileData {}
 
 /**
  * Type of the user profile labels structure (currently

src/core/packages/user-profile/common/src/user_profile.ts

@@ -10,7 +10,7 @@
 import { mockCoreContext } from '@kbn/core-base-server-mocks';
 import { httpServerMock } from '@kbn/core-http-server-mocks';
 import { userProfileServiceMock } from '@kbn/core-user-profile-server-mocks';
-import type { UserProfileWithSecurity } from '@kbn/core-user-profile-common';
+import type { UserProfileWithSecurity, UserProfileData } from '@kbn/core-user-profile-common';
 import { UserSettingsService } from './user_settings_service';
 
 describe('#setup', () => {
@@ -26,18 +26,18 @@ describe('#setup', () => {
     };
   });
 
-  const createUserProfile = (darkMode: string | undefined): UserProfileWithSecurity => {
+  const createUserProfile = (
+    userSettings: Partial<NonNullable<UserProfileData['userSettings']>>
+  ): UserProfileWithSecurity => {
     return {
       data: {
-        userSettings: {
-          darkMode,
-        },
+        userSettings,
       },
     } as unknown as UserProfileWithSecurity;
   };
 
   it('fetches userSettings when client is set and returns `true` when `darkMode` is set to `dark`', async () => {
-    startDeps.userProfile.getCurrent.mockResolvedValue(createUserProfile('dark'));
+    startDeps.userProfile.getCurrent.mockResolvedValue(createUserProfile({ darkMode: 'dark' }));
 
     const { getUserSettingDarkMode } = service.setup();
     service.start(startDeps);
@@ -54,7 +54,7 @@ describe('#setup', () => {
   });
 
   it('fetches userSettings when client is set and returns `false` when `darkMode` is set to `light`', async () => {
-    startDeps.userProfile.getCurrent.mockResolvedValue(createUserProfile('light'));
+    startDeps.userProfile.getCurrent.mockResolvedValue(createUserProfile({ darkMode: 'light' }));
 
     const { getUserSettingDarkMode } = service.setup();
     service.start(startDeps);
@@ -71,7 +71,7 @@ describe('#setup', () => {
   });
 
   it('fetches userSettings when client is set and returns `system` when `darkMode` is set to `system`', async () => {
-    startDeps.userProfile.getCurrent.mockResolvedValue(createUserProfile('system'));
+    startDeps.userProfile.getCurrent.mockResolvedValue(createUserProfile({ darkMode: 'system' }));
 
     const { getUserSettingDarkMode } = service.setup();
     service.start(startDeps);
@@ -88,7 +88,7 @@ describe('#setup', () => {
   });
 
   it('fetches userSettings when client is set and returns `undefined` when `darkMode` is set to `` (the default value)', async () => {
-    startDeps.userProfile.getCurrent.mockResolvedValue(createUserProfile(''));
+    startDeps.userProfile.getCurrent.mockResolvedValue(createUserProfile({ darkMode: undefined }));
 
     const { getUserSettingDarkMode } = service.setup();
     service.start(startDeps);
@@ -105,7 +105,9 @@ describe('#setup', () => {
   });
 
   it('fetches userSettings when client is set and returns `undefined` when `darkMode` is set to `space_default`', async () => {
-    startDeps.userProfile.getCurrent.mockResolvedValue(createUserProfile('space_default'));
+    startDeps.userProfile.getCurrent.mockResolvedValue(
+      createUserProfile({ darkMode: 'space_default' })
+    );
 
     const { getUserSettingDarkMode } = service.setup();
     service.start(startDeps);
@@ -121,6 +123,63 @@ describe('#setup', () => {
     });
   });
 
+  it('fetches userSettings when client is set and returns `true` when `rememberSelectedSpace` is set to `true`', async () => {
+    startDeps.userProfile.getCurrent.mockResolvedValue(
+      createUserProfile({ rememberSelectedSpace: true })
+    );
+
+    const { getUserSettingRememberSelectedSpace } = service.setup();
+    service.start(startDeps);
+
+    const kibanaRequest = httpServerMock.createKibanaRequest();
+    const rememberSelectedSpace = await getUserSettingRememberSelectedSpace(kibanaRequest);
+
+    expect(rememberSelectedSpace).toEqual(true);
+    expect(startDeps.userProfile.getCurrent).toHaveBeenCalledTimes(1);
+    expect(startDeps.userProfile.getCurrent).toHaveBeenCalledWith({
+      request: kibanaRequest,
+      dataPath: 'userSettings',
+    });
+  });
+
+  it('fetches userSettings when client is set and returns `false` when `rememberSelectedSpace` is set to `false`', async () => {
+    startDeps.userProfile.getCurrent.mockResolvedValue(
+      createUserProfile({ rememberSelectedSpace: false })
+    );
+
+    const { getUserSettingRememberSelectedSpace } = service.setup();
+    service.start(startDeps);
+
+    const kibanaRequest = httpServerMock.createKibanaRequest();
+    const rememberSelectedSpace = await getUserSettingRememberSelectedSpace(kibanaRequest);
+
+    expect(rememberSelectedSpace).toEqual(false);
+    expect(startDeps.userProfile.getCurrent).toHaveBeenCalledTimes(1);
+    expect(startDeps.userProfile.getCurrent).toHaveBeenCalledWith({
+      request: kibanaRequest,
+      dataPath: 'userSettings',
+    });
+  });
+
+  it('fetches userSettings when client is set and returns `true` when `rememberSelectedSpace` is not set', async () => {
+    startDeps.userProfile.getCurrent.mockResolvedValue(
+      createUserProfile({ rememberSelectedSpace: undefined })
+    );
+
+    const { getUserSettingRememberSelectedSpace } = service.setup();
+    service.start(startDeps);
+
+    const kibanaRequest = httpServerMock.createKibanaRequest();
+    const rememberSelectedSpace = await getUserSettingRememberSelectedSpace(kibanaRequest);
+
+    expect(rememberSelectedSpace).toEqual(true);
+    expect(startDeps.userProfile.getCurrent).toHaveBeenCalledTimes(1);
+    expect(startDeps.userProfile.getCurrent).toHaveBeenCalledWith({
+      request: kibanaRequest,
+      dataPath: 'userSettings',
+    });
+  });
+
   it('does not fetch userSettings when client is not set, returns `undefined`, and logs a debug statement', async () => {
     const { getUserSettingDarkMode } = service.setup();