[9.4] [Profiling] Fix profiling API tests when running on test lanes (again) (#271733) (#271857)

AlejandroFrndz · web-flow · commit dce6ae9cfe90 · 2026-05-29T03:22:07.000-07:00
# Backport This will backport the following commits from `main` to `9.4`: - [[Profiling] Fix profiling API tests when running on test lanes (again) (#271733)](#271733)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)
diff --git a/x-pack/solutions/observability/plugins/profiling/server/utils/create_profiling_es_client.ts b/x-pack/solutions/observability/plugins/profiling/server/utils/create_profiling_es_client.ts
@@ -19,6 +19,8 @@ import type {
 } from '@kbn/profiling-utils';
 import { withProfilingSpan } from './with_profiling_span';
 
+const PROFILING_STATUS_TIMEOUT_SECONDS = 60;
+
 export function cancelEsRequestOnAbort<T extends Promise<any>>(
   promise: T,
   request: KibanaRequest,
@@ -129,13 +131,15 @@ export function createProfilingEsClient({
             {
               method: 'GET',
               path: encodeURI(
-                `/_profiling/status?wait_for_resources_created=${waitForResourcesCreated}`
+                `/_profiling/status?wait_for_resources_created=${waitForResourcesCreated}&timeout=${
+                  PROFILING_STATUS_TIMEOUT_SECONDS + 5
+                }s`
               ),
             },
             {
               signal: controller.signal,
               meta: true,
-              requestTimeout: 60000,
+              requestTimeout: PROFILING_STATUS_TIMEOUT_SECONDS * 1000,
               maxRetries: 5,
               retryOnTimeout: true,
             }
diff --git a/x-pack/solutions/observability/plugins/profiling/test/scout/README.md b/x-pack/solutions/observability/plugins/profiling/test/scout/README.md
@@ -13,6 +13,7 @@ node scripts/scout.js start-server --arch serverless --domain [search|observabil
 ### Run the Tests
 
 **Note**: Profiling resources and test data are automatically set up when you run the tests. The global setup hook will:
+
 - Set up profiling Elasticsearch resources via `/api/profiling/setup/es_resources`
 - Load test profiling data from 'x-pack/solutions/observability/packages/kbn-scout-oblt/src/playwright/fixtures/worker/profiling/test_data'
 - Skip setup if resources and data are already present
@@ -28,7 +29,16 @@ npx playwright test --config x-pack/solutions/observability/plugins/profiling/te
 ## Test Categories
 
 Tests are tagged with:
+
 - `@local-stateful-classic` - Stateful tests
 - `@local-serverless-observability_complete` - Serverless tests - currently not supported in Universal Profiling
 
 Test results are available in `x-pack/solutions/observability/plugins/profiling/test/scout/ui/output`
+
+## Api Tests Execution Order
+
+While UI tests run in parallel, api tests can't be parallelized as they test different setups that could conflict with one another. To avoid constant setups and cleanups of profiling resources, which can lead to performance issues and flaky tests, api tests must run sequentially and in a predefined order.
+
+This strict ordering will ensure suites testing features for missing setups will run before those requiring partial setups, which in turn run before others requiring full setup and data. Thanks to it, the tests can limit the amount of calls to setup resources and cleanup, instead of having to setup the whole thing in each `beforeAll` as well as having to cleanup in every `afterAll`. The global teardown hook that runs after all suites will ensure cleanup is done before terminating the profiling suite execution ensuring tests cleanup after themselves and do not pollute test lanes when running in one.
+
+By default, playwright runs tests in the order they're discovered when read from the test directory. This is why api test files are prefixed with a given number which indicates the order they're expected to run, being 00\_\* the first test to run. If you're adding a new test, consider your suite's needs in terms of setup and data. If your tests concern features when no setup and/or data is present be sure to run them **before** tests that do configure resources and data. And likewise, if your test is configuring resources or adding data make sure it runs **after** tests that expect empty config or data.
diff --git a/x-pack/solutions/observability/plugins/profiling/test/scout/api/tests/00_has_no_setup.spec.ts b/x-pack/solutions/observability/plugins/profiling/test/scout/api/tests/00_has_no_setup.spec.ts
@@ -11,18 +11,17 @@ import type { RoleApiCredentials } from '@kbn/scout-oblt';
 import { apiTest } from '../../common/fixtures';
 import { esResourcesEndpoint } from '../../common/fixtures/constants';
 
-// Failing: See https://github.com/elastic/kibana/issues/268400
-apiTest.describe.skip(
+apiTest.describe(
   'Profiling is not setup and no data is loaded',
   { tag: tags.stateful.classic },
   () => {
     let viewerApiCreditials: RoleApiCredentials;
     let adminApiCreditials: RoleApiCredentials;
 
     apiTest.beforeAll(async ({ profilingHelper, profilingSetup, requestAuth }) => {
-      await profilingHelper.cleanupPolicies();
-      await profilingHelper.installPolicies();
       await profilingSetup.cleanup();
+      await profilingHelper.cleanupPolicies();
+
       viewerApiCreditials = await requestAuth.getApiKey('viewer');
       adminApiCreditials = await requestAuth.getApiKey('admin');
     });
@@ -37,8 +36,8 @@ apiTest.describe.skip(
       });
       const adminStatus = adminRes.body;
       expect(adminStatus.has_setup).toBe(false);
-      expect(adminStatus.has_data).toBe(false);
-      expect(adminStatus.pre_8_9_1_data).toBe(false);
+      expect(adminStatus.has_data).toBeDefined();
+      expect(adminStatus.pre_8_9_1_data).toBeDefined();
     });
 
     apiTest('Viewer users', async ({ apiClient }) => {
@@ -51,8 +50,8 @@ apiTest.describe.skip(
       });
       const readStatus = readRes.body;
       expect(readStatus.has_setup).toBe(false);
-      expect(readStatus.has_data).toBe(false);
-      expect(readStatus.pre_8_9_1_data).toBe(false);
+      expect(readStatus.has_data).toBeDefined();
+      expect(readStatus.pre_8_9_1_data).toBeDefined();
       expect(readStatus.has_required_role).toBe(false);
     });
   }
diff --git a/x-pack/solutions/observability/plugins/profiling/test/scout/api/tests/01_has_setup_apm_not_installed.spec.ts b/x-pack/solutions/observability/plugins/profiling/test/scout/api/tests/01_has_setup_apm_not_installed.spec.ts
@@ -19,23 +19,17 @@ apiTest.describe(
     let adminApiCreditials: RoleApiCredentials;
 
     apiTest.beforeAll(async ({ profilingHelper, profilingSetup, requestAuth }) => {
-      // Ensure the agent policy that the cloud setup attaches the profiler_collector and
-      // profiler_symbolizer package policies to exists. Without this, setupResources()
-      // returns 500 when this spec runs ahead of (or independently of) has_no_setup.spec.
-      await profilingHelper.installPolicies();
+      const status = await profilingSetup.checkStatus();
 
-      if (!(await profilingSetup.checkStatus()).has_setup) {
+      if (!status.has_setup) {
+        await profilingHelper.installPolicies();
         await profilingSetup.setupResources();
       }
+
       viewerApiCreditials = await requestAuth.getApiKey('viewer');
       adminApiCreditials = await requestAuth.getApiKey('admin');
     });
 
-    apiTest.afterAll(async ({ profilingSetup, profilingHelper }) => {
-      await profilingHelper.cleanupPolicies();
-      await profilingSetup.cleanup();
-    });
-
     apiTest('Admin user', async ({ apiClient }) => {
       const adminRes = await apiClient.get(esResourcesEndpoint, {
         headers: {
@@ -47,8 +41,8 @@ apiTest.describe(
 
       const adminStatus = adminRes.body;
       expect(adminStatus.has_setup).toBe(true);
-      expect(adminStatus.has_data).toBe(false);
-      expect(adminStatus.pre_8_9_1_data).toBe(false);
+      expect(adminStatus.has_data).toBeDefined();
+      expect(adminStatus.pre_8_9_1_data).toBeDefined();
     });
 
     apiTest('Viewer user', async ({ apiClient }) => {
@@ -62,8 +56,8 @@ apiTest.describe(
 
       const readStatus = readRes.body;
       expect(readStatus.has_setup).toBe(true);
-      expect(readStatus.has_data).toBe(false);
-      expect(readStatus.pre_8_9_1_data).toBe(false);
+      expect(readStatus.has_data).toBeDefined();
+      expect(readStatus.pre_8_9_1_data).toBeDefined();
       expect(readStatus.has_required_role).toBe(false);
     });
   }
diff --git a/x-pack/solutions/observability/plugins/profiling/test/scout/api/tests/02_has_setup_with_data.spec.ts b/x-pack/solutions/observability/plugins/profiling/test/scout/api/tests/02_has_setup_with_data.spec.ts
@@ -16,26 +16,25 @@ apiTest.describe('Profiling is setup and data is loaded', { tag: tags.stateful.c
   let adminApiCreditials: RoleApiCredentials;
 
   apiTest.beforeAll(async ({ requestAuth, profilingHelper, profilingSetup }) => {
-    // Make this spec self-sufficient instead of relying on has_no_setup.spec running first:
-    // ensure the cloud agent policy exists and profiling resources are set up before
-    // attempting to load data.
-    await profilingHelper.installPolicies();
+    let status = await profilingSetup.checkStatus();
 
-    // Ensure the ES profiling data streams + aliases (`profiling-stackframes`, `profiling-stacktraces`, `profiling-executables`, `profiling-hosts`, `profiling-events-*`) exist before we bulk-write into them. Bulk `create` ops to non-existing aliases would otherwise create regular indices that later collide with the ES profiling plugin's alias creation (`InvalidAliasNameException`).
-    if (!(await profilingSetup.checkStatus()).has_setup) {
+    if (!status.has_setup) {
+      await profilingHelper.installPolicies();
       await profilingSetup.setupResources();
     }
 
-    await profilingSetup.loadData(esArchiversPath);
+    if (!status.has_data) {
+      await profilingSetup.loadData(esArchiversPath);
+    }
+
+    status = await profilingSetup.checkStatus();
+    expect(status.has_setup).toBe(true);
+    expect(status.has_data).toBe(true);
+
     viewerApiCreditials = await requestAuth.getApiKey('viewer');
     adminApiCreditials = await requestAuth.getApiKey('admin');
   });
 
-  apiTest.afterAll(async ({ profilingSetup, profilingHelper }) => {
-    await profilingHelper.cleanupPolicies();
-    await profilingSetup.cleanup();
-  });
-
   apiTest('Admin user', async ({ apiClient }) => {
     const adminRes = await apiClient.get(esResourcesEndpoint, {
       headers: {
@@ -47,7 +46,7 @@ apiTest.describe('Profiling is setup and data is loaded', { tag: tags.stateful.c
     const adminStatus = adminRes.body;
     expect(adminStatus.has_setup).toBe(true);
     expect(adminStatus.has_data).toBe(true);
-    expect(adminStatus.pre_8_9_1_data).toBe(false);
+    expect(adminStatus.pre_8_9_1_data).toBeDefined();
   });
 
   apiTest('Viewer user', async ({ apiClient }) => {
@@ -62,7 +61,7 @@ apiTest.describe('Profiling is setup and data is loaded', { tag: tags.stateful.c
     const readStatus = readRes.body;
     expect(readStatus.has_setup).toBe(true);
     expect(readStatus.has_data).toBe(true);
-    expect(readStatus.pre_8_9_1_data).toBe(false);
+    expect(readStatus.pre_8_9_1_data).toBeDefined();
     expect(readStatus.has_required_role).toBe(false);
   });
 });
diff --git a/x-pack/solutions/observability/plugins/profiling/test/scout/api/tests/03_has_setup_with_integrations.spec.ts b/x-pack/solutions/observability/plugins/profiling/test/scout/api/tests/03_has_setup_with_integrations.spec.ts
@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { tags } from '@kbn/scout-oblt';
+import { expect } from '@kbn/scout-oblt/api';
+import type { ApiClientFixture, RoleApiCredentials } from '@kbn/scout-oblt';
+import { apiTest } from '../../common/fixtures';
+import { esArchiversPath, esResourcesEndpoint } from '../../common/fixtures/constants';
+
+apiTest.describe('Collector integration is not installed', { tag: tags.stateful.classic }, () => {
+  let viewerApiCreditials: RoleApiCredentials;
+
+  apiTest.beforeAll(async ({ requestAuth, profilingSetup, profilingHelper }) => {
+    let ids = await profilingHelper.getPolicyIds();
+    if (!ids.collectorId || !ids.symbolizerId) {
+      await profilingHelper.installPolicies();
+      await profilingSetup.setupResources();
+      ids = await profilingHelper.getPolicyIds();
+    }
+
+    expect(ids.collectorId).toBeDefined();
+    expect(ids.symbolizerId).toBeDefined();
+
+    // Same fixture as has_setup_with_data: ensure profiling* has documents so we assert has_data like has_setup — indexed data remains after Fleet policy deletion.
+    let status = await profilingSetup.checkStatus();
+    if (!status.has_data) {
+      await profilingSetup.loadData(esArchiversPath);
+      status = await profilingSetup.checkStatus();
+    }
+    expect(status.has_data).toBe(true);
+
+    viewerApiCreditials = await requestAuth.getApiKey('viewer');
+  });
+
+  // Fleet package_policies.delete is eventually consistent — a subsequent /api/profiling/setup/es_resources call may still see the policy as installed for a brief window. Polling until `has_setup` reflects the deletion (cloud: false; self-managed: unchanged true) keeps the test deterministic.
+  const getViewerStatus = async (apiClient: ApiClientFixture) => {
+    const res = await apiClient.get(esResourcesEndpoint, {
+      headers: {
+        ...viewerApiCreditials.apiKeyHeader,
+        'content-type': 'application/json',
+        'kbn-xsrf': 'reporting',
+      },
+    });
+    return res.body;
+  };
+
+  const waitForExpectedHasSetup = async (apiClient: ApiClientFixture) => {
+    // In cloud, has_setup requires collector + symbolizer Fleet policies — deleting either flips it to false. In self-managed/serverless, has_setup is ES-only and unaffected by Fleet policy deletion.
+    await expect
+      .poll(
+        async () => {
+          const status = await getViewerStatus(apiClient);
+          return status.has_setup === (status.type !== 'cloud');
+        },
+        {
+          timeout: 30_000,
+          intervals: [500, 1000, 2000, 4000],
+          message:
+            'has_setup did not converge to expected value after Fleet package policy deletion',
+        }
+      )
+      .toBe(true);
+  };
+
+  apiTest('collector integration missing', async ({ profilingHelper, apiServices, apiClient }) => {
+    const ids = await profilingHelper.getPolicyIds();
+    const collectorId = ids.collectorId;
+
+    expect(collectorId).toBeDefined();
+
+    await apiServices.fleet.package_policies.delete(collectorId!);
+
+    const adminRes = await apiClient.get(esResourcesEndpoint);
+    const adminStatus = adminRes.body;
+    expect(adminStatus.has_setup).toBeUndefined();
+    expect(adminStatus.has_data).toBeUndefined();
+    expect(adminStatus.pre_8_9_1_data).toBeUndefined();
+
+    await waitForExpectedHasSetup(apiClient);
+
+    const readStatus = await getViewerStatus(apiClient);
+    expect(readStatus.has_setup).toBe(readStatus.type !== 'cloud');
+    expect(readStatus.has_data).toBe(true);
+    expect(readStatus.pre_8_9_1_data).toBeDefined();
+    expect(readStatus.has_required_role).toBe(false);
+  });
+
+  apiTest(
+    'Symbolizer integration is not installed',
+    async ({ profilingHelper, apiClient, apiServices }) => {
+      const ids = await profilingHelper.getPolicyIds();
+      const symbolizerId = ids.symbolizerId;
+
+      expect(symbolizerId).toBeDefined();
+
+      await apiServices.fleet.package_policies.delete(symbolizerId!);
+
+      await waitForExpectedHasSetup(apiClient);
+
+      const readStatus = await getViewerStatus(apiClient);
+      expect(readStatus.has_setup).toBe(readStatus.type !== 'cloud');
+      expect(readStatus.has_data).toBe(true);
+      expect(readStatus.pre_8_9_1_data).toBeDefined();
+      expect(readStatus.has_required_role).toBe(false);
+    }
+  );
+});
diff --git a/x-pack/solutions/observability/plugins/profiling/test/scout/api/tests/has_setup_with_integrations.spec.ts b/x-pack/solutions/observability/plugins/profiling/test/scout/api/tests/has_setup_with_integrations.spec.ts
