[scout] re-org test directory (elastic#226411)

## Summary

Since Scout will soon support `api` testing, we need to re-org scout
test directory so that both ui and api playwright configs can be
discovered and executed independently. Proposed structure:

```
- plugin
 |- test
   |- scout
     |- common // fixtures/constants to share across both 'ui' and 'api' tests
     |- ui
        |-tests
        |-parallel_tests
        |-playwright.config.ts
        |-playwright.parallel.config.ts
     |- api
        |-tests
        |-parallel_tests
        |-playwright.config.ts
        |-playwright.parallel.config.ts
```

This PR updates all the existing scout directories

---------

Co-authored-by: kibanamachine <[email protected]>

Author: dmlemeshko

.eslintrc.js

@@ -1596,9 +1596,9 @@ module.exports = {
         'src/platform/packages/shared/kbn-scout/src/playwright/**/*.ts',
         'x-pack/solutions/observability/packages/kbn-scout-oblt/src/playwright/**/*.ts',
         'x-pack/solutions/security/packages/kbn-scout-security/src/playwright/**/*.ts',
-        'src/platform/plugins/**/ui_tests/**/*.ts',
-        'x-pack/platform/plugins/**/ui_tests/**/*.ts',
-        'x-pack/solutions/**/plugins/**/ui_tests/**/*.ts',
+        'src/platform/plugins/**/test/scout/**/*.ts',
+        'x-pack/platform/plugins/**/test/scout/**/*.ts',
+        'x-pack/solutions/**/plugins/**/test/scout/**/*.ts',
       ],
       excludedFiles: ['src/platform/packages/shared/kbn-scout/src/playwright/**/*.test.ts'],
       extends: ['plugin:playwright/recommended'],
@@ -2319,8 +2319,8 @@ module.exports = {
     },
     {
       files: [
-        'src/platform/plugins/**/ui_tests/**/*.ts',
-        'x-pack/platform/**/plugins/**/ui_tests/**/*.ts',
+        'src/platform/plugins/**/test/scout/**/*.ts',
+        'x-pack/platform/**/plugins/**/test/scout/**/*.ts',
       ],
       rules: {
         'no-restricted-imports': [
@@ -2341,7 +2341,7 @@ module.exports = {
       },
     },
     {
-      files: ['x-pack/solutions/observability/plugins/**/ui_tests/**/*.ts'],
+      files: ['x-pack/solutions/observability/plugins/**/test/scout/**/*.ts'],
       rules: {
         'no-restricted-imports': [
           'error',
@@ -2368,7 +2368,7 @@ module.exports = {
       },
     },
     {
-      files: ['x-pack/solutions/security/plugins/**/ui_tests/**/*.ts'],
+      files: ['x-pack/solutions/security/plugins/**/test/scout/**/*.ts'],
       rules: {
         'no-restricted-imports': [
           'error',

.github/CODEOWNERS

@@ -1584,7 +1584,7 @@ x-pack/solutions/observability/plugins/observability/server/lib/esql_extensions
 /src/platform/test/functional/apps/discover/observability/embeddable @elastic/obs-ux-logs-team
 /src/platform/test/functional/apps/discover/observability/logs @elastic/obs-ux-logs-team
 /x-pack/solutions/observability/plugins/observability/public/pages/landing @elastic/obs-ux-logs-team
-/x-pack/solutions/observability/plugins/observability/ui_tests/tests/landing.spec.ts @elastic/obs-ux-logs-team
+/x-pack/solutions/observability/plugins/observability/test/scout/ui/tests/landing.spec.ts @elastic/obs-ux-logs-team
 
 # Observability-ui
 /x-pack/test_serverless/api_integration/test_suites/observability/index.ts @elastic/observability-ui
@@ -1744,10 +1744,10 @@ x-pack/platform/plugins/shared/ml/server/models/data_recognizer/modules/security
 /src/platform/packages/shared/kbn-es/src/serverless_resources/project_roles/security/roles.yml @elastic/appex-qa
 /x-pack/platform/test/common/ftr_provider_context.ts @elastic/appex-qa
 /x-pack/platform/test/common/lib/test_data_loader.ts @elastic/appex-qa # temporarily
-/x-pack/platform/plugins/shared/maps/ui_tests @elastic/appex-qa # temporarily
-/x-pack/platform/plugins/private/discover_enhanced/ui_tests/ @elastic/appex-qa # temporarily
-/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/discover_cdp_perf.spec.ts @elastic/kibana-data-discovery # test tracks bundle size limits
-/x-pack/platform/plugins/private/painless_lab/ui_tests @elastic/appex-qa # temporarily
+/x-pack/platform/plugins/shared/maps/test/scout @elastic/appex-qa # temporarily
+/x-pack/platform/plugins/private/discover_enhanced/test/scout/ @elastic/appex-qa # temporarily
+/x-pack/platform/plugins/private/discover_enhanced/test/scout/ui/tests/discover_cdp_perf.spec.ts @elastic/kibana-data-discovery # test tracks bundle size limits
+/x-pack/platform/plugins/private/painless_lab/test/scout @elastic/appex-qa # temporarily
 /x-pack/test/functional/fixtures/package_registry_config.yml @elastic/appex-qa # No usages found
 /x-pack/test/functional/fixtures/kbn_archiver/packaging.json @elastic/appex-qa # No usages found
 /x-pack/test/functional/es_archives/filebeat @elastic/appex-qa

packages/kbn-repo-source-classifier/src/config.ts

@@ -58,6 +58,5 @@ export const TEST_DIR = new Set([
   'storybook',
   '.storybook',
   'integration_tests',
-  'ui_tests',
   ...RANDOM_TEST_FILE_NAMES,
 ]);

src/platform/packages/private/kbn-scout-reporting/src/helpers/plugin_manifest.test.ts

@@ -16,15 +16,15 @@ jest.mock('fs');
 describe('plugin_manifest', () => {
   describe('getManifestPath', () => {
     it('should resolve the manifest path correctly for a valid config path', () => {
-      const configPath = '/plugins/my_plugin/ui_tests/playwright.config.ts';
+      const configPath = '/plugins/my_plugin/test/scout/ui/playwright.config.ts';
       const expectedPath = path.resolve('/plugins/my_plugin/kibana.jsonc');
       expect(getManifestPath(configPath)).toBe(expectedPath);
     });
 
-    it(`should throw an error if 'ui_tests' is not in the path`, () => {
+    it(`should throw an error if 'scout' is not in the path`, () => {
       const configPath = '/plugins/my_plugin/tests/playwright.config.ts';
       expect(() => getManifestPath(configPath)).toThrow(
-        /Invalid path: "ui_tests" directory not found/
+        /Invalid path: "scout" directory not found/
       );
     });
   });
@@ -90,7 +90,7 @@ describe('plugin_manifest', () => {
   });
 
   describe('getPluginManifestData', () => {
-    const configPath = '/plugins/my_plugin/ui_tests/playwright.config.ts';
+    const configPath = '/plugins/my_plugin/test/scout/ui/playwright.config.ts';
     const manifestContent = `
       {
         "id": "my_plugin",

src/platform/packages/private/kbn-scout-reporting/src/helpers/plugin_manifest.ts

@@ -23,20 +23,20 @@ export interface PluginManifest {
  * Resolves the path to the `kibana.jsonc` manifest based on the Playwright configuration file path.
  * @param configPath - Absolute path to the Playwright configuration file.
  * @returns Absolute path to the `kibana.jsonc` file.
- * @throws Error if `ui_tests` is not found in the path.
+ * @throws Error if `scout` is not found in the path.
  */
 export const getManifestPath = (configPath: string): string => {
   const pathSegments = configPath.split(path.sep);
-  const testDirIndex = pathSegments.indexOf('ui_tests');
+  const testDirIndex = pathSegments.indexOf('scout');
 
   if (testDirIndex === -1) {
     throw new Error(
-      `Invalid path: "ui_tests" directory not found in ${configPath}.
-  Ensure playwright configuration file is in the plugin directory: '/plugins/<plugin-name>/ui_tests/<config-file>'`
+      `Invalid path: "scout" directory not found in ${configPath}.
+  Ensure playwright configuration file is in the plugin directory: '/plugins/<plugin-name>/test/scout/ui/<config-file>'`
     );
   }
 
-  const manifestSegments = pathSegments.slice(0, testDirIndex).concat('kibana.jsonc');
+  const manifestSegments = pathSegments.slice(0, testDirIndex - 1).concat('kibana.jsonc');
   return path.resolve('/', ...manifestSegments); // Ensure absolute path
 };
 

src/platform/packages/shared/kbn-scout/README.md

@@ -84,8 +84,8 @@ import { createPlaywrightConfig } from '@kbn/scout';
 
 // eslint-disable-next-line import/no-default-export
 export default createPlaywrightConfig({
-    testDir: './tests',
-    workers: 2,
+  testDir: './tests',
+  workers: 2,
 });
 ```
 
@@ -100,6 +100,7 @@ The `fixtures` directory contains core Scout capabilities required for testing t
 scoped to either `test` or `worker`. Scope decides when to init a new fixture instance: once per worker or for every test function. It is important to choose the correct scope to keep test execution optimally fast: if **a new instance is not needed for every test**, the fixture should be scoped to **worker**. Otherwise, it should be scoped to **test**.
 
 **Core `worker` scoped fixtures:**
+
 - `log`
 - `config`
 - `esClient`
@@ -108,20 +109,21 @@ scoped to either `test` or `worker`. Scope decides when to init a new fixture in
 - `samlAuth`
 
 ```ts
-    test.beforeAll(async ({ kbnClient }) => {
-      await kbnClient.importExport.load(testData.KBN_ARCHIVES.ECOMMERCE);
-    });
+test.beforeAll(async ({ kbnClient }) => {
+  await kbnClient.importExport.load(testData.KBN_ARCHIVES.ECOMMERCE);
+});
 ```
 
 **Core `test` scoped fixtures:**
+
 - `browserAuth`
 - `pageObjects`
 - `page`
 
 ```ts
-    test.beforeEach(async ({ browserAuth }) => {
-      await browserAuth.loginAsViewer();
-    });
+test.beforeEach(async ({ browserAuth }) => {
+  await browserAuth.loginAsViewer();
+});
 ```
 
 If a new fixture depends on a fixture with a `test` scope, it must also be `test` scoped.
@@ -148,7 +150,7 @@ All registered Page Objects are available via the `pageObjects` fixture:
 
 ```ts
 test.beforeEach(async ({ pageObjects }) => {
-    await pageObjects.discover.goto();
+  await pageObjects.discover.goto();
 });
 ```
 
@@ -219,7 +221,7 @@ This command is useful for manual testing or running tests via an IDE.
 To start the servers locally and run tests in one step, use:
 
 ```bash
-node scripts/scout.js run-tests [--stateful|--serverless=[es|oblt|security]] --config <plugin-path>/ui_tests/playwright.config.ts
+node scripts/scout.js run-tests [--stateful|--serverless=[es|oblt|security]] --config <plugin-path>/test/scout/ui/playwright.config.ts
 ```
 
 - **`--stateful`** or **`--serverless`**: Specifies the deployment type.
@@ -235,7 +237,7 @@ If the servers are already running, you can execute tests independently using on
 2. **Command Line**: Use the following command:
 
 ```bash
-npx playwright test --config <plugin-path>/ui_tests/playwright.config.ts --project local
+npx playwright test --config <plugin-path>/test/scout/ui/playwright.config.ts --project local
 ```
 
 - **`--project`**: Specifies the test target as `local` ( `ech` or `mki` for Cloud targets, see below).
@@ -250,14 +252,14 @@ To run tests against a Cloud deployment, you can use either the Scout CLI or the
 node scripts/scout.js run-tests \
   --stateful \
   --testTarget=cloud \
-  --config <plugin-path>/ui_tests/playwright.config.ts
+  --config <plugin-path>/test/scout/ui/playwright.config.ts
 ```
 
 ```bash
 node scripts/scout.js run-tests \
   --serverless=oblt \
   --testTarget=cloud \
-  --config <plugin-path>/ui_tests/playwright.config.ts
+  --config <plugin-path>/test/scout/ui/playwright.config.ts
 ```
 
 - **`--testTarget=cloud`**: Specifies that tests should run against a Cloud deployment.
@@ -268,14 +270,14 @@ node scripts/scout.js run-tests \
 npx playwright test \
   --project=ech \
   --grep=@ess \
-  --config <plugin-path>/ui_tests/playwright.config.ts
+  --config <plugin-path>/test/scout/ui/playwright.config.ts
 ```
 
 ```bash
 npx playwright test \
   --project=mki \
   --grep=@svlOblt \
-  --config <plugin-path>/ui_tests/playwright.config.ts
+  --config <plugin-path>/test/scout/ui/playwright.config.ts
 ```
 
 - **`--project`**: Specifies the test target (`ech` for Stateful or `mki` for Serverless).
@@ -298,6 +300,7 @@ node scripts/jest --config src/platform/packages/shared/kbn-scout/jest.config.js
 Ensure you have the latest local copy of the Kibana repository.
 
 Install dependencies by running the following commands:
+
 - `yarn kbn bootstrap` to install dependencies.
 - `node scripts/build_kibana_platform_plugins.js` to build plugins.
 
@@ -317,7 +320,9 @@ export class NewPage {
   // implementation
 }
 ```
+
 2. **Register the Page Object:** Update the index file to include the new Page Object:
+
 ```ts
 export function createCorePageObjects(page: ScoutPage): PageObjects {
   return {
@@ -326,6 +331,7 @@ export function createCorePageObjects(page: ScoutPage): PageObjects {
   };
 }
 ```
+
 #### Adding API service
 
 1. **Create a New API service:** Add your service to the `src/playwright/fixtures/worker/apis` directory. For instance:
@@ -351,7 +357,9 @@ export const getFleetApiHelper = (log: ScoutLogger, kbnClient: KbnClient): Fleet
   };
 };
 ```
+
 2. **Register the API service:** Update the index file to include the new service:
+
 ```ts
 export const apiServicesFixture = coreWorkerFixtures.extend<
   {},
@@ -367,7 +375,9 @@ export const apiServicesFixture = coreWorkerFixtures.extend<
   ],
 });
 ```
+
 #### Adding Fixture
+
 1. **Determine Fixture Scope:** Decide if your fixture should apply to the `test` (per-test) or `worker` (per-worker) scope.
 
 2. **Implement the Fixture:** Add the implementation to `src/playwright/fixtures/test` or `src/playwright/fixtures/worker`.
@@ -376,20 +386,20 @@ export const apiServicesFixture = coreWorkerFixtures.extend<
 export const newTestFixture = base.extend<ScoutTestFixtures, ScoutWorkerFixtures>({
   newFixture: async ({}, use) => {
     const myFn = // implementation
-    await use(myFn);
+      await use(myFn);
     // optionally, cleanup on test completion
   },
 });
 ```
+
 3. **Register the Fixture:** Add the fixture to the appropriate scope:
+
 ```ts
-export const scoutTestFixtures = mergeTests(
-  ...
-  newTestFixture,
-);
+export const scoutTestFixtures = mergeTests(coreFixtures, newTestFixture);
 ```
 
 #### Best Practices
+
 - **Reusable Code:** When creating Page Objects, API services or Fixtures that apply to more than one plugin, ensure they are added to the `kbn-scout` package.
 - **Adhere to Existing Structure:** Maintain consistency with the project's architecture.
 - **Keep the Scope of Components Clear** When designing test components, keep in naming conventions, scope, maintainability and performance.
@@ -399,30 +409,32 @@ export const scoutTestFixtures = mergeTests(
 - **Add Unit Tests:** Include tests for new logic where applicable, ensuring it works as expected.
 - **Playwright documentation:** [Official best practices](https://playwright.dev/docs/best-practices)
 
-
 ### Running tests on CI
 
 #### Enabling tests for execution
+
 Scout is still in active development, which means frequent code changes may sometimes cause test failures. To maintain stability, we currently do not run Scout tests for every PR and encourage teams to limit the number of tests they add for now.
 
 If a test is difficult to stabilize within a reasonable timeframe, we reserve the right to disable it or even all tests for particular plugin.
 
 To manage Scout test execution, we use the `.buildkite/scout_ci_config.yml` file, where Kibana plugins with Scout tests are registered. If you're unsure about the stability of your tests, please add your plugin under the `disabled` section.
 
 You can check whether your plugin is already registered by running:
+
 ```bash
 node scripts/scout discover-playwright-configs --validate
 ```
+
 On CI we run Scout tests only for `enabled` plugins:
 
 For PRs, Scout tests run only if there are changes to registered plugins or Scout-related packages.
 On merge commits, Scout tests run in a non-blocking mode.
 
 #### Scout exit codes
 
-| Exit code | Description |
-|--------|--------|
-| 0 | All tests passed |
-| 1 | Missing configuration (e.g. SCOUT_CONFIG_GROUP_KEY and SCOUT_CONFIG_GROUP_TYPE environment variables not set) |
-| 2 | No tests in Playwright config | 
-| 10| Tests failed | 
+| Exit code | Description                                                                                                   |
+| --------- | ------------------------------------------------------------------------------------------------------------- |
+| 0         | All tests passed                                                                                              |
+| 1         | Missing configuration (e.g. SCOUT_CONFIG_GROUP_KEY and SCOUT_CONFIG_GROUP_TYPE environment variables not set) |
+| 2         | No tests in Playwright config                                                                                 |
+| 10        | Tests failed                                                                                                  |

src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.test.ts

@@ -37,32 +37,32 @@ describe('getScoutPlaywrightConfigs', () => {
 
   it('should correctly extract plugin names and group config files', () => {
     (fastGlob.sync as jest.Mock).mockReturnValue([
-      'x-pack/platform/plugins/private/plugin_a/ui_tests/playwright.config.ts',
-      'x-pack/platform/plugins/private/plugin_a/ui_tests/parallel.playwright.config.ts',
-      'x-pack/solutions/security/plugins/plugin_b/ui_tests/playwright.config.ts',
-      'src/platform/plugins/shared/plugin_c/ui_tests/playwright.config.ts',
+      'x-pack/platform/plugins/private/plugin_a/test/scout/ui/playwright.config.ts',
+      'x-pack/platform/plugins/private/plugin_a/test/scout/ui/parallel.playwright.config.ts',
+      'x-pack/solutions/security/plugins/plugin_b/test/scout/ui/playwright.config.ts',
+      'src/platform/plugins/shared/plugin_c/test/scout/ui/playwright.config.ts',
     ]);
 
     const plugins = getScoutPlaywrightConfigs(['x-pack/', 'src/'], mockLog);
 
     expect(plugins.size).toBe(3);
     expect(plugins.get('plugin_a')).toEqual({
       configs: [
-        'x-pack/platform/plugins/private/plugin_a/ui_tests/playwright.config.ts',
-        'x-pack/platform/plugins/private/plugin_a/ui_tests/parallel.playwright.config.ts',
+        'x-pack/platform/plugins/private/plugin_a/test/scout/ui/playwright.config.ts',
+        'x-pack/platform/plugins/private/plugin_a/test/scout/ui/parallel.playwright.config.ts',
       ],
       usesParallelWorkers: true,
       group: 'platform',
       pluginPath: 'x-pack/platform/plugins/private/plugin_a',
     });
     expect(plugins.get('plugin_b')).toEqual({
-      configs: ['x-pack/solutions/security/plugins/plugin_b/ui_tests/playwright.config.ts'],
+      configs: ['x-pack/solutions/security/plugins/plugin_b/test/scout/ui/playwright.config.ts'],
       usesParallelWorkers: false,
       group: 'security',
       pluginPath: 'x-pack/solutions/security/plugins/plugin_b',
     });
     expect(plugins.get('plugin_c')).toEqual({
-      configs: ['src/platform/plugins/shared/plugin_c/ui_tests/playwright.config.ts'],
+      configs: ['src/platform/plugins/shared/plugin_c/test/scout/ui/playwright.config.ts'],
       usesParallelWorkers: false,
       group: 'platform',
       pluginPath: 'src/platform/plugins/shared/plugin_c',