Decentralize synthtrace scenarios: colocate with plugin code (#270157)

## Summary

Implements Phase 1 and Phase 2 of #259332.

### Phase 1 — Enable colocated scenarios (non-breaking)

- Removes the `choices` restriction from `run_synthtrace.ts` so the CLI
accepts any file path (absolute or relative), not just built-in scenario
names
- Exports `Scenario`, `ScenarioInitOptions`, `RunOptions`,
`getSynthtraceEnvironment`, `ComponentTemplateName`,
`IndexTemplateName`, `timestampDateMapping`, and `withClient` from the
`@kbn/synthtrace` public API, so plugins can define colocated scenarios
without importing from internal paths

### Phase 2 — Move scenarios to their respective plugins

| Destination | Scenarios moved |
|---|---|
| `x-pack/solutions/observability/plugins/apm/test/scenarios/` | ~40 APM
scenarios (traces, services, metrics, OTel, mobile, Lambda, etc.) +
APM-specific helpers |
| `x-pack/solutions/observability/plugins/infra/test/scenarios/` | 6
infra scenarios (`infra_aws_rds`, `infra_docker_containers`,
`infra_hosts_ecs`, `infra_hosts_semconv`, `infra_k8s_containers`,
`infra_k8s_pods`) |
| `x-pack/solutions/observability/plugins/synthetics/test/scenarios/` |
`degraded_synthetics_monitors` |
| `x-pack/platform/plugins/shared/streams/test/scenarios/` | 14
logs/HTTP scenarios + logs/HTTP helpers |
| Kept in `@kbn/synthtrace` | 7 multi-domain cross-cutting scenarios
(`logs_and_metrics*`, `logs_traces_hosts`,
`kubernetes_logs_traces_pods`, `infra_hosts_*_with_apm_hosts`,
`distributed_unstructured_logs`) + `sre_incidents/` + `sigevents/`
subdirs |

Each destination plugin's `tsconfig.json` is updated to include
`test/scenarios/**/*` and add `@kbn/synthtrace` to `kbn_references`. An
`.eslintrc.js` disabling `import/no-default-export` is added to each
scenarios directory to allow the `export default` pattern required by
the `Scenario` type.

### Usage after this PR

```bash
# Colocated scenario by path
node scripts/synthtrace.js ./x-pack/solutions/observability/plugins/apm/test/scenarios/distributed_trace.ts

# Built-in scenario name still works (backward compatible)
node scripts/synthtrace.js simple_trace
```

## Test plan

- [ ] `node scripts/check_changes.ts` passes
- [ ] Type check passes for `@kbn/synthtrace`, `apm`, `infra`,
`synthetics`, and `streams` packages
- [ ] `node scripts/synthtrace.js simple_trace` still works (backward
compat)
- [ ] `node scripts/synthtrace.js
./x-pack/solutions/observability/plugins/apm/test/scenarios/distributed_trace.ts`
works with a colocated path

Closes #259332

Made with [Cursor](https://cursor.com)

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: 4 people

src/platform/packages/shared/kbn-synthtrace/EXAMPLES.md

@@ -93,21 +93,21 @@ const esEvents = toElasticsearchOutput([
 For live data ingestion:
 
 ```sh
-node scripts/synthtrace simple_trace.ts --target=http://admin:changeme@localhost:9200 --live
+node scripts/synthtrace simple_trace --target=http://admin:changeme@localhost:9200 --live
 ```
 
 For a fixed time window:
 
 ```sh
-node scripts/synthtrace simple_trace.ts --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now
+node scripts/synthtrace simple_trace --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now
 ```
 
 ### Local Development
 
 When running the CLI locally, you can simply use the following command to ingest data to a locally running Elasticsearch and Kibana instance:
 
 ```sh
-node scripts/synthtrace simple_trace.ts
+node scripts/synthtrace simple_trace
 ```
 
 _Assuming both Elasticsearch and Kibana are running on the default localhost ports with default credentials._
@@ -119,23 +119,23 @@ If the Kibana URL differs from the Elasticsearch URL in protocol or hostname, yo
 For example when running ES (with ssl) and Kibana (without ssl) locally in Serverless mode:
 
 ```sh
-node scripts/synthtrace simple_trace.ts --target=https://elastic_serverless:changeme@localhost:9200 --kibana=http://elastic_serverless:changeme@localhost:5601
+node scripts/synthtrace simple_trace --target=https://elastic_serverless:changeme@localhost:9200 --kibana=http://elastic_serverless:changeme@localhost:5601
 ```
 
 ### Using CLI for Elastic Cloud URLs
 
 If you are ingesting data to Elastic Cloud, you can pass the `--target` option with the Elastic Cloud URL:
 
 ```sh
-node scripts/synthtrace simple_trace.ts --target=https://<username>:<password>@your-cloud-cluster.kb.us-west2.gcp.elastic-cloud.com/
+node scripts/synthtrace simple_trace --target=https://<username>:<password>@your-cloud-cluster.kb.us-west2.gcp.elastic-cloud.com/
 ```
 
 ### Using CLI with an API key
 
 You can use a Kibana API key for authentication by passing the `--apiKey` option:
 
 ```sh
-node scripts/synthtrace simple_trace.ts --target=https://my-deployment.es.us-central1.gcp.elastic.cloud --apiKey="your-api-key"
+node scripts/synthtrace simple_trace --target=https://my-deployment.es.us-central1.gcp.elastic.cloud --apiKey="your-api-key"
 ```
 
 ## Scenario Examples

src/platform/packages/shared/kbn-synthtrace/README.md

@@ -115,13 +115,13 @@ Via the CLI, you can run scenarios, either using a fixed time range or continuou
 For live data ingestion:
 
 ```sh
-node scripts/synthtrace simple_trace.ts --target=http://admin:changeme@localhost:9200 --live
+node scripts/synthtrace simple_trace --target=http://admin:changeme@localhost:9200 --live
 ```
 
 For a fixed time window:
 
 ```sh
-node scripts/synthtrace simple_trace.ts --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now
+node scripts/synthtrace simple_trace --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now
 ```
 
 The script will try to automatically find bootstrapped APM indices. **If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.**
@@ -131,7 +131,7 @@ The script will try to automatically find bootstrapped APM indices. **If these i
 When running the CLI locally, you can simply use the following command to ingest data to a locally running Elasticsearch and Kibana instance:
 
 ```sh
-node scripts/synthtrace simple_trace.ts
+node scripts/synthtrace simple_trace
 ```
 
 _Assuming both Elasticsearch and Kibana are running on the default localhost ports with default credentials._
@@ -144,7 +144,7 @@ For example when running ES (with ssl) and Kibana (without ssl) locally in Serve
 Also use `localhost` instead of `127.0.0.1` as the hostname as `127.0.0.1` will likely not work with self-signed certificates.
 
 ```sh
-node scripts/synthtrace simple_trace.ts --target=https://elastic_serverless:changeme@localhost:9200 --kibana=http://elastic_serverless:changeme@localhost:5601
+node scripts/synthtrace simple_trace --target=https://elastic_serverless:changeme@localhost:9200 --kibana=http://elastic_serverless:changeme@localhost:5601
 ```
 
 #### Using CLI for Elastic Cloud URLs
@@ -153,15 +153,15 @@ If you are ingesting data to Elastic Cloud, you can pass the `--target` option w
 Or you can pass only `--kibana` and the CLI will infer the Elasticsearch URL from the Kibana URL. Or pass both if URLs are not in default scheme.
 
 ```sh
-node scripts/synthtrace simple_trace.ts --target=https://<username>:<password>@your-cloud-cluster.kb.us-west2.gcp.elastic-cloud.com/
+node scripts/synthtrace simple_trace --target=https://<username>:<password>@your-cloud-cluster.kb.us-west2.gcp.elastic-cloud.com/
 ```
 
 #### Using CLI with an API key
 
 You can use a Kibana API key for authentication by passing the `--apiKey` option. When the `--apiKey` is provided, it will be used for authentication with both Elasticsearch and Kibana, taking precedence over other types of authentication.
 
 ```sh
-node scripts/synthtrace simple_trace.ts --target=https://my-deployment.es.us-central1.gcp.elastic.cloud --apiKey="your-api-key"
+node scripts/synthtrace simple_trace --target=https://my-deployment.es.us-central1.gcp.elastic.cloud --apiKey="your-api-key"
 ```
 
 For more CLI examples, see [Examples](./EXAMPLES.md#cli-examples).

src/platform/packages/shared/kbn-synthtrace/index.ts

@@ -21,18 +21,31 @@ export {
   type GetClientsReturn,
 } from './src/cli/utils/clients_manager';
 export type { ApmSynthtraceEsClient } from './src/lib/apm/client/apm_synthtrace_es_client';
+export { ComponentTemplateName } from './src/lib/apm/client/apm_synthtrace_es_client';
 export type { InfraSynthtraceEsClient } from './src/lib/infra/infra_synthtrace_es_client';
 export type { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client';
 export type { SyntheticsSynthtraceEsClient } from './src/lib/synthetics/synthetics_synthtrace_es_client';
 export {
   addObserverVersionTransform,
   deleteSummaryFieldTransform,
 } from './src/lib/utils/transform_helpers';
-export { indexAll } from './src/lib/utils/with_client';
+export { indexAll, withClient } from './src/lib/utils/with_client';
+export type { Scenario, ScenarioInitOptions } from './src/cli/scenario';
+export type { RunOptions } from './src/cli/utils/parse_run_cli_flags';
+export { getSynthtraceEnvironment } from './src/lib/utils/get_synthtrace_environment';
+export { IndexTemplateName } from './src/lib/logs/custom_logsdb_index_templates';
+export { timestampDateMapping } from './src/lib/logs/custom_logsdb_indices';
 export * from './src/scenarios/agent_builder';
 
 export { sigEvents } from './src/lib/service_graph_logs';
 
+export {
+  getNumberOpt,
+  getBooleanOpt,
+  getStringOpt,
+} from './src/scenarios/helpers/scenario_opts_helpers';
+export { getAtIndexOrRandom } from './src/scenarios/helpers/get_at_index_or_random';
+
 export {
   APM_METRICS_SERVICE_NAMES,
   APM_METRICS_DASHBOARD_SERVICES,

src/platform/packages/shared/kbn-synthtrace/moon.yml

@@ -22,7 +22,6 @@ dependsOn:
   - '@kbn/dev-utils'
   - '@kbn/elastic-agent-utils'
   - '@kbn/zod'
-  - '@kbn/sample-log-parser'
   - '@kbn/tooling-log'
   - '@kbn/streams-schema'
   - '@kbn/streamlang'

src/platform/packages/shared/kbn-synthtrace/src/cli/run_synthtrace.ts

@@ -10,25 +10,18 @@
 import datemath from '@kbn/datemath';
 import type { Argv } from 'yargs';
 import yargs from 'yargs/yargs';
-import { readdirSync } from 'fs';
-import path from 'path';
 import { intervalToMs } from './utils/interval_to_ms';
 import { parseRunCliFlags } from './utils/parse_run_cli_flags';
 import { startHistoricalDataUpload } from './utils/start_historical_data_upload';
 import { startLiveDataUpload } from './utils/start_live_data_upload';
 
-function getBuiltinScenarios() {
-  return readdirSync(path.resolve(__dirname, '../scenarios')).map((s) => s.replace(/\.ts$/, ''));
-}
-
 function options(y: Argv) {
   return y
     .usage('$0 <files ...>')
     .positional('files', {
-      describe: 'Name of scenario',
+      describe: 'Name of a built-in scenario or a path (absolute or relative) to a scenario file',
       demandOption: true,
       string: true,
-      choices: getBuiltinScenarios(),
     })
     .option('target', {
       describe: 'Elasticsearch target',
@@ -145,6 +138,10 @@ function options(y: Argv) {
       '$0 simple_logs --target=https://elastic:changeme@localhost:9200 --insecure',
       'Connect to HTTPS Elasticsearch with self-signed certificates'
     )
+    .example(
+      '$0 ./x-pack/solutions/observability/plugins/apm/test/scenarios/distributed_trace.ts',
+      'Run a colocated scenario by path'
+    )
     .showHelpOnFail(false)
     .wrap(null);
 }

src/platform/packages/shared/kbn-synthtrace/src/cli/utils/parse_run_cli_flags.ts

@@ -12,6 +12,109 @@ import { pick } from 'lodash';
 import path from 'path';
 import { LogLevel } from '../../lib/utils/create_logger';
 import type { RunCliFlags } from '../run_synthtrace';
+import { discoverRepoRoot } from './read_kibana_config';
+
+const REPO_ROOT = discoverRepoRoot();
+
+/**
+ * Maps short scenario names to their colocated plugin paths.
+ * Populated when scenarios were moved out of @kbn/synthtrace in
+ * https://github.com/elastic/kibana/pull/270157.
+ */
+const SCENARIO_ALIASES: Record<string, string> = {
+  agent_config: 'x-pack/solutions/observability/plugins/apm/test/scenarios/agent_config.ts',
+  apache_logs: 'x-pack/platform/plugins/shared/streams/test/scenarios/apache_logs.ts',
+  apm_jvm_metrics_type_conflict:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/apm_jvm_metrics_type_conflict.ts',
+  apm_metrics_dashboards:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/apm_metrics_dashboards.ts',
+  apm_ml_anomalies: 'x-pack/solutions/observability/plugins/apm/test/scenarios/apm_ml_anomalies.ts',
+  apm_service_legacy_to_otel_metrics:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/apm_service_legacy_to_otel_metrics.ts',
+  apm_service_multi_env_otel_migration:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/apm_service_multi_env_otel_migration.ts',
+  apm_service_overlapping_otel_metrics:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/apm_service_overlapping_otel_metrics.ts',
+  aws_lambda: 'x-pack/solutions/observability/plugins/apm/test/scenarios/aws_lambda.ts',
+  azure_functions: 'x-pack/solutions/observability/plugins/apm/test/scenarios/azure_functions.ts',
+  cloud_services_icons:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/cloud_services_icons.ts',
+  complete_trace: 'x-pack/solutions/observability/plugins/apm/test/scenarios/complete_trace.ts',
+  composite_spans: 'x-pack/solutions/observability/plugins/apm/test/scenarios/composite_spans.ts',
+  continuous_rollups:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/continuous_rollups.ts',
+  degraded_logs: 'x-pack/platform/plugins/shared/streams/test/scenarios/degraded_logs.ts',
+  degraded_synthetics_monitors:
+    'x-pack/solutions/observability/plugins/synthetics/test/scenarios/degraded_synthetics_monitors.ts',
+  diagnostic_service_map:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/diagnostic_service_map.ts',
+  distributed_trace:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/distributed_trace.ts',
+  distributed_trace_long:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/distributed_trace_long.ts',
+  error_with_missing_transaction_sampled:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/error_with_missing_transaction_sampled.ts',
+  failed_logs: 'x-pack/platform/plugins/shared/streams/test/scenarios/failed_logs.ts',
+  high_throughput: 'x-pack/solutions/observability/plugins/apm/test/scenarios/high_throughput.ts',
+  http_access_logs: 'x-pack/platform/plugins/shared/streams/test/scenarios/http_access_logs.ts',
+  http_access_logs_otel:
+    'x-pack/platform/plugins/shared/streams/test/scenarios/http_access_logs_otel.ts',
+  infra_aws_rds: 'x-pack/solutions/observability/plugins/infra/test/scenarios/infra_aws_rds.ts',
+  infra_docker_containers:
+    'x-pack/solutions/observability/plugins/infra/test/scenarios/infra_docker_containers.ts',
+  infra_hosts_ecs: 'x-pack/solutions/observability/plugins/infra/test/scenarios/infra_hosts_ecs.ts',
+  infra_hosts_semconv:
+    'x-pack/solutions/observability/plugins/infra/test/scenarios/infra_hosts_semconv.ts',
+  infra_k8s_containers:
+    'x-pack/solutions/observability/plugins/infra/test/scenarios/infra_k8s_containers.ts',
+  infra_k8s_pods: 'x-pack/solutions/observability/plugins/infra/test/scenarios/infra_k8s_pods.ts',
+  kafka_topics: 'x-pack/solutions/observability/plugins/apm/test/scenarios/kafka_topics.ts',
+  kubernetes_logs: 'x-pack/platform/plugins/shared/streams/test/scenarios/kubernetes_logs.ts',
+  low_throughput: 'x-pack/solutions/observability/plugins/apm/test/scenarios/low_throughput.ts',
+  many_dependencies:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/many_dependencies.ts',
+  many_errors: 'x-pack/solutions/observability/plugins/apm/test/scenarios/many_errors.ts',
+  many_instances: 'x-pack/solutions/observability/plugins/apm/test/scenarios/many_instances.ts',
+  many_otel_services:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/many_otel_services.ts',
+  many_services: 'x-pack/solutions/observability/plugins/apm/test/scenarios/many_services.ts',
+  many_transactions:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/many_transactions.ts',
+  messaging_systems_mixed:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/messaging_systems_mixed.ts',
+  missing_service_environment:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/missing_service_environment.ts',
+  mobile: 'x-pack/solutions/observability/plugins/apm/test/scenarios/mobile.ts',
+  otel_exit_span_missing_destination:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/otel_exit_span_missing_destination.ts',
+  otel_logs_and_metrics_only:
+    'x-pack/platform/plugins/shared/streams/test/scenarios/otel_logs_and_metrics_only.ts',
+  otel_simple_trace:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/otel_simple_trace.ts',
+  other_bucket_group:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/other_bucket_group.ts',
+  sample_logs: 'x-pack/platform/plugins/shared/streams/test/scenarios/sample_logs.ts',
+  serverless_logs: 'x-pack/platform/plugins/shared/streams/test/scenarios/serverless_logs.ts',
+  service_map: 'x-pack/solutions/observability/plugins/apm/test/scenarios/service_map.ts',
+  service_map_oom: 'x-pack/solutions/observability/plugins/apm/test/scenarios/service_map_oom.ts',
+  service_summary_field_version_dependent:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/service_summary_field_version_dependent.ts',
+  services_without_transactions:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/services_without_transactions.ts',
+  simple_logs: 'x-pack/platform/plugins/shared/streams/test/scenarios/simple_logs.ts',
+  simple_non_ecs_logs:
+    'x-pack/platform/plugins/shared/streams/test/scenarios/simple_non_ecs_logs.ts',
+  simple_otel_logs: 'x-pack/platform/plugins/shared/streams/test/scenarios/simple_otel_logs.ts',
+  simple_trace: 'x-pack/solutions/observability/plugins/apm/test/scenarios/simple_trace.ts',
+  slash_logs: 'x-pack/platform/plugins/shared/streams/test/scenarios/slash_logs.ts',
+  span_links: 'x-pack/solutions/observability/plugins/apm/test/scenarios/span_links.ts',
+  trace_with_orphan_items:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/trace_with_orphan_items.ts',
+  trace_with_service_names_with_slashes:
+    'x-pack/solutions/observability/plugins/apm/test/scenarios/trace_with_service_names_with_slashes.ts',
+  unstructured_logs: 'x-pack/platform/plugins/shared/streams/test/scenarios/unstructured_logs.ts',
+  variance: 'x-pack/solutions/observability/plugins/apm/test/scenarios/variance.ts',
+};
 
 function getParsedFiles(flags: RunCliFlags) {
   const { _: parsedFiles } = flags;
@@ -21,13 +124,20 @@ function getParsedFiles(flags: RunCliFlags) {
   }
 
   const filesPath = parsedFiles.map((parsedFile) => {
-    const foundPath = [
-      path.resolve(parsedFile),
-      path.resolve(`${parsedFile}.ts`),
+    const aliasRelPath = SCENARIO_ALIASES[parsedFile];
+    const candidates = [
+      // Alias lookup for scenarios moved out of @kbn/synthtrace to plugin directories.
+      ...(aliasRelPath ? [path.resolve(REPO_ROOT, aliasRelPath)] : []),
+      // Built-in scenarios that remain in @kbn/synthtrace.
       path.resolve(__dirname, '../../scenarios', parsedFile),
       path.resolve(__dirname, '../../scenarios', `${parsedFile}.ts`),
       path.resolve(__dirname, '../../scenarios', `${parsedFile}.js`),
-    ].find((p) => existsSync(p));
+      // Absolute or CWD-relative path provided directly.
+      path.resolve(parsedFile),
+      path.resolve(`${parsedFile}.ts`),
+    ];
+
+    const foundPath = candidates.find((p) => existsSync(p));
 
     if (!foundPath) {
       throw new Error(`Could not find scenario file for: "${parsedFile}"`);

src/platform/packages/shared/kbn-synthtrace/src/cli/utils/read_kibana_config.ts

@@ -12,6 +12,8 @@ import fs from 'fs';
 import yaml from 'js-yaml';
 import { REPO_ROOT } from '@kbn/repo-info';
 
+export const discoverRepoRoot = (): string => REPO_ROOT;
+
 export type KibanaConfig = ReturnType<typeof readKibanaConfig>;
 
 export const readKibanaConfig = () => {

src/platform/packages/shared/kbn-synthtrace/src/cli/utils/workers/run_worker.ts

@@ -47,7 +47,7 @@ export function runWorker<TWorkerData>({
 
     worker.on('error', (error) => {
       logger.error(error);
-      reject();
+      reject(error);
     });
 
     worker.on('exit', (code) => {

src/platform/packages/shared/kbn-synthtrace/src/scenarios/helpers/get_at_index_or_random.ts

@@ -9,6 +9,13 @@
 
 import { randomInt } from 'crypto';
 
-// Utility function to get a random element from an array
-export const getAtIndexOrRandom = <T>(values: T[], index?: number) =>
-  values[index ?? randomInt(values.length)];
+/**
+ * Returns `values[index]` when index is provided, otherwise picks a uniformly random element.
+ * Throws if the array is empty to surface programming errors early.
+ */
+export const getAtIndexOrRandom = <T>(values: T[], index?: number): T => {
+  if (values.length === 0) {
+    throw new Error('getAtIndexOrRandom called with an empty array');
+  }
+  return values[index ?? randomInt(values.length)];
+};