[9.3] [profiling] fix failing profiling tests due to [fleet-agent-policies/policy-elastic-agent-on-cloud] not found (#268428)

x-pack/solutions/observability/packages/kbn-scout-oblt/src/playwright/fixtures/worker/profiling/profiling_setup_fixture.ts

@@ -47,10 +47,20 @@ export const profilingSetupFixture = base.extend<{}, { profilingSetup: Profiling
               'content-type': 'application/json',
               'kbn-xsrf': 'reporting',
             },
+            // The route is not idempotent on retry; we want a fast, clear failure so the
+            // underlying error (logged in Kibana by handleRouteHandlerError) is the first
+            // thing the developer sees instead of a series of 500s.
+            retries: 0,
           });
           log.info('Profiling resources set up successfully');
-        } catch (error) {
-          log.error(`Error setting up profiling resources: ${error}`);
+        } catch (error: any) {
+          const status = error?.response?.status ?? error?.originalError?.response?.status;
+          const body = error?.response?.data ?? error?.originalError?.response?.data;
+          log.error(
+            `Error setting up profiling resources POST /api/profiling/setup/es_resources: status=${status} body=${JSON.stringify(
+              body
+            )}`
+          );
           throw error;
         }
       };
@@ -97,20 +107,25 @@ export const profilingSetupFixture = base.extend<{}, { profilingSetup: Profiling
       const cleanup = async (): Promise<void> => {
         log.info(`Unloading Profiling data`);
 
-        const indices = await esClient.cat.indices({ format: 'json' });
+        const ignoreNotFound = (error: any) => {
+          if (error?.meta?.statusCode === 404) return undefined;
+          throw error;
+        };
 
-        const profilingIndices = indices
-          .filter((index) => index.index !== undefined)
+        const profilingIndices = (await esClient.cat.indices({ format: 'json' }))
           .map((index) => index.index)
-          .filter((index) => {
-            return index!.startsWith('profiling') || index!.startsWith('.profiling');
-          }) as string[];
+          .filter(
+            (name): name is string =>
+              !!name && (name.startsWith('profiling') || name.startsWith('.profiling'))
+          );
 
         await Promise.all([
-          ...profilingIndices.map((index) => esClient.indices.delete({ index })),
-          esClient.indices.deleteDataStream({
-            name: 'profiling-events*',
-          }),
+          ...profilingIndices.map((index) =>
+            esClient.indices.delete({ index, ignore_unavailable: true }).catch(ignoreNotFound)
+          ),
+          // 'profiling-events*' may not exist on a fresh cluster; tolerate the 404 so we
+          // do not abort the whole Promise.all and leave preceding cleanup work half-done.
+          esClient.indices.deleteDataStream({ name: 'profiling-events*' }).catch(ignoreNotFound),
         ]);
         log.info('Unloaded Profiling data');
       };

x-pack/solutions/observability/plugins/profiling/test/scout/api/tests/has_setup_apm_not_installed.spec.ts

@@ -17,7 +17,12 @@ apiTest.describe(
   () => {
     let viewerApiCreditials: RoleApiCredentials;
     let adminApiCreditials: RoleApiCredentials;
-    apiTest.beforeAll(async ({ profilingSetup, requestAuth }) => {
+    apiTest.beforeAll(async ({ profilingHelper, profilingSetup, requestAuth }) => {
+      // Ensure the agent policy that the cloud setup attaches the profiler_collector and
+      // profiler_symbolizer package policies to exists. Without this, setupResources()
+      // returns 500 when this spec runs ahead of (or independently of) has_no_setup.spec.
+      await profilingHelper.installPolicies();
+
       if (!(await profilingSetup.checkStatus()).has_setup) {
         await profilingSetup.setupResources();
       }

x-pack/solutions/observability/plugins/profiling/test/scout/api/tests/has_setup_with_data.spec.ts

@@ -14,7 +14,16 @@ import { esArchiversPath, esResourcesEndpoint } from '../../common/fixtures/cons
 apiTest.describe('Profiling is setup and data is loaded', { tag: tags.stateful.classic }, () => {
   let viewerApiCreditials: RoleApiCredentials;
   let adminApiCreditials: RoleApiCredentials;
-  apiTest.beforeAll(async ({ requestAuth, profilingSetup }) => {
+  apiTest.beforeAll(async ({ requestAuth, profilingHelper, profilingSetup }) => {
+    // Make this spec self-sufficient instead of relying on has_no_setup.spec running first:
+    // ensure the cloud agent policy exists and profiling resources are set up before
+    // attempting to load data.
+    await profilingHelper.installPolicies();
+
+    if (!(await profilingSetup.checkStatus()).has_setup) {
+      await profilingSetup.setupResources();
+    }
+
     await profilingSetup.loadData(esArchiversPath);
     viewerApiCreditials = await requestAuth.getApiKey('viewer');
     adminApiCreditials = await requestAuth.getApiKey('admin');