Merge branch 'main' into fix-uri-parts-usage-string

Author: diogolima-elastic

.agents/skills/evals-write-spec/SKILL.md

@@ -24,7 +24,7 @@ evaluate.describe('Suite name', { tag: tags.serverless.observability.complete },
 
   evaluate('test name', async ({ executorClient, connector }) => {
     await executorClient.runExperiment(
-      { dataset, task },
+      { datasets: [dataset], task },
       evaluators
     );
   });
@@ -202,7 +202,7 @@ export function createEvaluateDataset({
   return async ({ dataset }) => {
     await executorClient.runExperiment(
       {
-        dataset,
+        datasets: [dataset],
         task: async ({ input }) => {
           const response = await chatClient.converse({ messages: [{ message: input.question }] });
           return { messages: response.messages, steps: response.steps };

.agents/skills/evals-write-spec/references/evaluator-patterns.md

@@ -20,7 +20,7 @@ type MyTaskOutput = TaskOutput & {
 };
 
 await executorClient.runExperiment(
-  { dataset, task },
+  { datasets: [dataset], task },
   selectEvaluators<MyExample, MyTaskOutput>([
     {
       name: 'NonEmptyDocuments',
@@ -246,7 +246,7 @@ A common pattern passes both CODE and LLM evaluators to `runExperiment`:
 
 ```ts
 await executorClient.runExperiment(
-  { dataset, task },
+  { datasets: [dataset], task },
   [
     createCriteriaEvaluator({ evaluators }),
     createToolCallsEvaluator({ evaluators }),

.buildkite/scripts/common/setup_job_env.sh

@@ -130,7 +130,7 @@ EOF
 # Set up Kibana Evals secrets
 {
   if [[ "${KBN_EVALS:-}" =~ ^(1|true)$ ]]; then
-    echo "KBN_EVALS was set - exposing evals connectors and ES export credentials"
+    echo "KBN_EVALS was set - exposing evals connectors and export credentials"
 
     KBN_EVALS_CONFIG_JSON="$(vault_get kbn-evals config | base64 -d)"
     # Validate config shape (safe; does not print secrets)
@@ -145,10 +145,6 @@ EOF
     export KBN_EVALS_CONFIG_B64
     KBN_EVALS_CONFIG_B64="$(printf '%s' "$KBN_EVALS_CONFIG_JSON" | base64)"
 
-    # Elasticsearch cluster for evaluation results export
-    export EVALUATIONS_ES_URL="$(jq -r '.evaluationsEs.url // empty' <<<"$KBN_EVALS_CONFIG_JSON")"
-    export EVALUATIONS_ES_API_KEY="$(jq -r '.evaluationsEs.apiKey // empty' <<<"$KBN_EVALS_CONFIG_JSON")"
-
     # Optional: separate cluster for trace-based evaluators
     export TRACING_ES_URL="$(jq -r '.tracingEs.url // empty' <<<"$KBN_EVALS_CONFIG_JSON")"
     export TRACING_ES_API_KEY="$(jq -r '.tracingEs.apiKey // empty' <<<"$KBN_EVALS_CONFIG_JSON")"

.buildkite/scripts/steps/evals/run_suite.sh

@@ -18,10 +18,10 @@ fi
 # The value should be the platform-level `pluginId` use-case identifier.
 # `@kbn/evals` defaults this to `kbn_evals`, but you can override via KBN_EVALS_TELEMETRY_PLUGIN_ID.
 
-# Set a base run id from the Buildkite build. The evaluator fixture appends the
-# connector id to produce a unique run_id per model (e.g. bk-<build>-<connector>).
-# Correlation across models in the same build uses ci.buildkite.build_id which
-# is populated automatically from BUILDKITE_BUILD_ID in score_repository.ts.
+# Set a base build run ID from the Buildkite build. This is used as a seed for
+# generating deterministic per-task experiment IDs (not as the experiment_id itself).
+# Suite-run grouping in the UI uses metadata.ci.build_id which is populated
+# automatically from BUILDKITE_BUILD_ID in the Buildkite metadata.
 if [[ -z "${TEST_RUN_ID:-}" ]] && [[ -n "${BUILDKITE_BUILD_ID:-}" ]]; then
   export TEST_RUN_ID="bk-${BUILDKITE_BUILD_ID}"
 fi

docs/extend/plugin-list.md

@@ -153,7 +153,7 @@ mapped_pages:
 | [enterpriseSearch](https://github.com/elastic/kibana/blob/main/x-pack/solutions/search/plugins/enterprise_search/README.md) | This plugin provides Kibana user interfaces for managing the Enterprise Search solution and its products, App Search and Workplace Search. |
 | [entityManager](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/entity_manager/README.md) | This plugin provides access to observed entity data, such as information about hosts, pods, containers, services, and more. |
 | [entityStore](https://github.com/elastic/kibana/blob/main/x-pack/solutions/security/plugins/entity_store/README.md) | Central place for Entities management and logs extraction. |
-| [evals](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/evals/README.md) | The Evals plugin provides an in-Kibana UI for browsing LLM evaluation run results, per-evaluator statistics, and OpenTelemetry traces produced by the @kbn/evals evaluation framework. |
+| [evals](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/evals/README.md) | The Evals plugin provides an in-Kibana UI for browsing LLM evaluation experiment results, per-evaluator statistics, and OpenTelemetry traces produced by the @kbn/evals evaluation framework. |
 | [eventLog](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/event_log/README.md) | The event log plugin provides a persistent history of alerting and action activities. |
 | [exploratoryView](https://github.com/elastic/kibana/blob/main/x-pack/solutions/observability/plugins/exploratory_view/README.md) | A shared component for visualizing observability data types via lens embeddable. For further details. |
 | [features](https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/features/README.md) | The features plugin enhance Kibana with a per-feature privilege system. |

packages/kbn-mock-idp-plugin/public/role_switcher.tsx

@@ -33,16 +33,16 @@ export const useAuthenticator = (reloadPage = false) => {
       body: JSON.stringify(params),
     });
 
-    if (reloadPage) {
-      const form = createForm(
-        services.http.basePath.prepend('/api/security/saml/callback'),
-        response
-      );
+    const { acsUrl, ...samlFields } = response;
+    const formAction = acsUrl ?? services.http.basePath.prepend('/api/security/saml/callback');
+
+    if (reloadPage || acsUrl) {
+      const form = createForm(formAction, samlFields);
       form.submit();
       await new Promise(() => {});
     } else {
       await services.http.post('/api/security/saml/callback', {
-        body: JSON.stringify(response),
+        body: JSON.stringify(samlFields),
         asResponse: true,
         rawResponse: true,
       });

packages/kbn-mock-idp-plugin/server/plugin.ts

@@ -201,6 +201,13 @@ export const plugin: PluginInitializer<void, void, PluginSetupDependencies> = as
             const parsed = new URL(request.body.url, 'https://localhost');
             const relayState = parsed.searchParams.get('RelayState') ?? undefined;
 
+            // Kibana-bound ACS URLs are intentionally left to the `onPreResponse` rewrite above;
+            // we only override here for external SPs (e.g. UIAM).
+            const externalAcsUrl =
+              samlRequestInfo?.acsUrl && !samlRequestInfo.acsUrl.startsWith(MOCK_IDP_SP_BASE_URL)
+                ? samlRequestInfo.acsUrl
+                : undefined;
+
             return response.ok({
               body: {
                 SAMLResponse: await createSAMLResponse({
@@ -212,9 +219,13 @@ export const plugin: PluginInitializer<void, void, PluginSetupDependencies> = as
                     ? { authnRequestId: samlRequestInfo.requestId }
                     : {}),
                   ...(samlRequestInfo?.issuer ? { spEntityId: samlRequestInfo.issuer } : {}),
+                  ...(externalAcsUrl ? { acsUrl: externalAcsUrl } : {}),
                   ...serverlessOptions,
                 }),
                 ...(relayState ? { RelayState: relayState } : {}),
+                // Echoed alongside SAMLResponse so the browser's auto-submitted form posts to UIAM
+                // instead of the default Kibana ACS endpoint (see mock_idp_page form action).
+                ...(externalAcsUrl ? { acsUrl: externalAcsUrl } : {}),
               },
             });
           } catch (err) {

packages/kbn-optimizer/limits.yml

@@ -169,7 +169,7 @@ pageLoadAssetSize:
   securitySolutionEss: 38689
   securitySolutionServerless: 52082
   serverless: 7412
-  serverlessObservability: 19300
+  serverlessObservability: 21437
   serverlessSearch: 26287
   serverlessVectordb: 7618
   serverlessWorkplaceAI: 4855

scripts/evals.js

@@ -274,20 +274,15 @@ var ENV_DOCS = [
     example: 'TRACING_EXPORTERS=\'[{"http":{"url":"https://ingest.example.com/v1/traces"}}]\'',
   },
   {
-    name: 'EVALUATIONS_ES_URL',
-    description: 'Elasticsearch URL where evaluation results are exported.',
-    example: 'EVALUATIONS_ES_URL=http://elastic:changeme@localhost:9200',
-  },
-  {
-    name: 'EVALUATIONS_ES_API_KEY',
-    description: 'API key for authenticating with the evaluations Elasticsearch cluster.',
-    example: 'EVALUATIONS_ES_API_KEY=...',
+    name: 'EVALUATIONS_KBN_URL',
+    description:
+      'Kibana URL used for eval score ingestion and dataset operations when targeting a non-local cluster.',
+    example: 'EVALUATIONS_KBN_URL=http://elastic:changeme@localhost:5601',
   },
   {
-    name: 'KBN_EVALS_SKIP_PREFLIGHT_EXPORT',
-    description:
-      'Skip the Elasticsearch export preflight check (not recommended for CI). Preflight runs a small sentinel write against the configured evaluations cluster.',
-    example: 'KBN_EVALS_SKIP_PREFLIGHT_EXPORT=true',
+    name: 'EVALUATIONS_KBN_API_KEY',
+    description: 'API key for authenticating to EVALUATIONS_KBN_URL.',
+    example: 'EVALUATIONS_KBN_API_KEY=...',
   },
   {
     name: 'SELECTED_EVALUATORS',
@@ -348,7 +343,7 @@ function runFastHelp() {
   logInfo('  stop [--service <name>]       Stop backgrounded eval services');
   logInfo('  logs [--service <name>]       Tail logs from eval services');
   logInfo('  scout                         Start Scout server for evals');
-  logInfo('  clear-index                   Delete kibana-evaluations indices (reset export)');
+  logInfo('  clear-index                   Delete .evaluation-scores indices (reset export)');
   logInfo('  run [--suite <id>] [...]      Run an eval suite');
   logInfo('  list [--refresh] [--json]     List eval suites');
   logInfo('  labels [suite-id ...]         Create/sync GitHub eval suite labels');

src/cli/serve/serve.js

@@ -462,6 +462,7 @@ function tryConfigureServerlessSamlProvider(rawConfig, opts, extraCliOptions) {
   // Ensure the plugin is loaded in dynamically to exclude from production build
   const {
     MOCK_IDP_REALM_NAME,
+    MOCK_IDP_UIAM_OAUTH_BASE_URL,
     MOCK_IDP_UIAM_SERVICE_URL,
     MOCK_IDP_UIAM_SHARED_SECRET,
     MOCK_IDP_UIAM_ORGANIZATION_ID,
@@ -515,6 +516,14 @@ function tryConfigureServerlessSamlProvider(rawConfig, opts, extraCliOptions) {
     lodashSet(rawConfig, 'xpack.security.uiam.ssl.verificationMode', 'none');
     lodashSet(rawConfig, 'mockIdpPlugin.uiam.enabled', true);
 
+    // SAML POST binding submits the response cross-origin to UIAM's ACS endpoint, so the
+    // enforced `form-action` directive (default `'self'`) needs to allow the UIAM origin.
+    const uiamOAuthOrigin = new url.URL(MOCK_IDP_UIAM_OAUTH_BASE_URL).origin;
+    const existingFormAction = _.get(rawConfig, 'csp.form_action', []);
+    if (!existingFormAction.includes(uiamOAuthOrigin)) {
+      lodashSet(rawConfig, 'csp.form_action', [...existingFormAction, uiamOAuthOrigin]);
+    }
+
     if (!_.has(rawConfig, 'xpack.security.uiam.url')) {
       lodashSet(rawConfig, 'xpack.security.uiam.url', MOCK_IDP_UIAM_SERVICE_URL);
     }