[Inference] Evaluations framework (elastic#227453)

introduces `@kbn/evals` to enable offline evaluation suites for
LLM-based workflows in Kibana, using:

- `@kbn/scout` and Playwright for running the evaluations
- `@kbn/inference-plugin` for handling LLM interactions
- [Phoenix](https://github.com/Arize-ai/phoenix) for storing evaluations

See:
https://github.com/dgieselaar/kibana/blob/evaluation-runner/x-pack/platform/packages/shared/kbn-evals/README.md

Other changes in this PR:

- Implement support for `exports` in package.json in
`@kbn/import-resolver` (this causes some eslint-disables to no longer be
necessary)
- Graceful shutdown for OpenTelemetry span processors
- Add `scripts/playwright.js` script which sets up babel transforms,
tracing etc

Notes:
- used o3 to implement support for `exports` (and then iterated on it)
- used o3 to generate a
[README](https://github.com/dgieselaar/kibana/blob/evaluation-runner/x-pack/platform/packages/shared/kbn-evals/README.md)
(and made some minor changes)

Dependencies:
- Added @arizeai/phoenix-client to interact with Phoenix's API. The
alternative is using REST/GraphQL directly but there's a lot of
client-side logic in the SDK.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Tiago Costa <tiago.costa@elastic.co>
Co-authored-by: Tiago Costa <tiagoffcc@hotmail.com>
Co-authored-by: Viduni Wickramarachchi <viduni.ushanka@gmail.com>

Author: 4 people

.eslintrc.js

@@ -795,6 +795,7 @@ module.exports = {
         'x-pack/test/profiling_api_integration/**/*.ts',
         'x-pack/test/security_solution_api_integration/*/test_suites/**/*',
         'x-pack/test/security_solution_api_integration/**/config*.ts',
+        '**/playwright.config.ts',
       ],
       rules: {
         'import/no-default-export': 'off',

.github/CODEOWNERS

@@ -869,6 +869,7 @@ x-pack/platform/packages/shared/kbn-elastic-assistant @elastic/security-generati
 x-pack/platform/packages/shared/kbn-elastic-assistant-common @elastic/security-generative-ai
 x-pack/platform/packages/shared/kbn-elastic-assistant-shared-state @elastic/security-generative-ai
 x-pack/platform/packages/shared/kbn-entities-schema @elastic/obs-entities
+x-pack/platform/packages/shared/kbn-evals @elastic/appex-ai-infra
 x-pack/platform/packages/shared/kbn-event-stacktrace @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team
 x-pack/platform/packages/shared/kbn-inference-cli @elastic/appex-ai-infra
 x-pack/platform/packages/shared/kbn-inference-endpoint-ui-common @elastic/appex-ai-infra
@@ -1051,6 +1052,7 @@ x-pack/solutions/observability/packages/alert-details @elastic/obs-ux-management
 x-pack/solutions/observability/packages/alerting-test-data @elastic/obs-ux-management-team
 x-pack/solutions/observability/packages/get-padded-alert-time-range-util @elastic/obs-ux-management-team
 x-pack/solutions/observability/packages/kbn-alerts-grouping @elastic/response-ops
+x-pack/solutions/observability/packages/kbn-evals-suite-obs-ai-assistant @elastic/obs-ai-assistant
 x-pack/solutions/observability/packages/kbn-genai-cli @elastic/obs-knowledge-team
 x-pack/solutions/observability/packages/kbn-observability-schema @elastic/obs-ux-management-team
 x-pack/solutions/observability/packages/kbn-scout-oblt @elastic/appex-qa

.gitignore

@@ -157,6 +157,9 @@ role_users.json
 # ignore Scout temp directory
 .scout
 
+# Playwright
+**/test-results/.last-run.json
+
 .devcontainer/.env
 
 # Ignore temporary files in oas_docs

package.json

@@ -1134,6 +1134,7 @@
     "@opentelemetry/exporter-trace-otlp-http": "^0.203.0",
     "@opentelemetry/exporter-trace-otlp-proto": "^0.203.0",
     "@opentelemetry/instrumentation": "^0.203.0",
+    "@opentelemetry/instrumentation-http": "^0.203.0",
     "@opentelemetry/instrumentation-undici": "^0.14.0",
     "@opentelemetry/otlp-exporter-base": "^0.203.0",
     "@opentelemetry/semantic-conventions": "^1.36.0",
@@ -1397,6 +1398,7 @@
   },
   "devDependencies": {
     "@apidevtools/swagger-parser": "^12.0.0",
+    "@arizeai/phoenix-client": "^2.3.4",
     "@babel/cli": "^7.24.7",
     "@babel/core": "^7.24.7",
     "@babel/eslint-parser": "^7.24.7",
@@ -1542,6 +1544,8 @@
     "@kbn/eslint-plugin-i18n": "link:packages/kbn-eslint-plugin-i18n",
     "@kbn/eslint-plugin-imports": "link:packages/kbn-eslint-plugin-imports",
     "@kbn/eslint-plugin-telemetry": "link:packages/kbn-eslint-plugin-telemetry",
+    "@kbn/evals": "link:x-pack/platform/packages/shared/kbn-evals",
+    "@kbn/evals-suite-obs-ai-assistant": "link:x-pack/solutions/observability/packages/kbn-evals-suite-obs-ai-assistant",
     "@kbn/expect": "link:src/platform/packages/shared/kbn-expect",
     "@kbn/failed-test-reporter-cli": "link:packages/kbn-failed-test-reporter-cli",
     "@kbn/find-used-node-modules": "link:packages/kbn-find-used-node-modules",
@@ -1942,6 +1946,7 @@
     "recast": "^0.23.9",
     "regenerate": "^1.4.0",
     "resolve": "^1.22.0",
+    "resolve.exports": "^2.0.3",
     "rxjs-marbles": "^7.0.1",
     "sass-embedded": "^1.79.6",
     "sass-loader": "^10.5.2",

renovate.json

@@ -197,7 +197,8 @@
         "cache-manager",
         "cache-manager-fs-hash",
         "keyv",
-        "@types/cache-manager-fs-hash"
+        "@types/cache-manager-fs-hash",
+        "@arizeai/phoenix-client"
       ],
       "reviewers": [
         "team:appex-ai-infra"
@@ -1932,6 +1933,25 @@
       "minimumReleaseAge": "7 days",
       "enabled": true
     },
+    {
+      "groupName": "resolve.exports",
+      "matchDepNames": [
+        "resolve.exports"
+      ],
+      "reviewers": [
+        "team:kibana-operations"
+      ],
+      "matchBaseBranches": [
+        "main"
+      ],
+      "labels": [
+        "Team:Operations",
+        "backport:all-open",
+        "release_note:skip"
+      ],
+      "minimumReleaseAge": "7 days",
+      "enabled": true
+    },
     {
       "groupName": "mocha",
       "matchDepNames": [
@@ -4357,6 +4377,7 @@
         "@opentelemetry/exporter-trace-otlp-http",
         "@opentelemetry/exporter-trace-otlp-proto",
         "@opentelemetry/instrumentation",
+        "@opentelemetry/instrumentation-http",
         "@opentelemetry/instrumentation-undici",
         "@opentelemetry/otlp-exporter-base",
         "@opentelemetry/semantic-conventions"

…out/src/playwright/global_hooks/index.ts scripts/playwright.jssrc/platform/packages/shared/kbn-scout/src/playwright/global_hooks/index.ts renamed to scripts/playwright.js src/platform/packages/shared/kbn-scout/src/playwright/global_hooks/index.ts renamed to scripts/playwright.js

@@ -7,5 +7,5 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-export { ingestTestDataHook } from './data_ingestion';
-export { ingestSynthtraceDataHook } from './synthtrace_ingestion';
+require('./setup_playwright');
+require('@playwright/test/cli');

scripts/setup_playwright.js

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+/**
+ * This file is intended to be required before Playwright runs. It is responsible
+ * for:
+ * - Calling setup_node_env which registers babel transforms, source map support,
+ * hardening etc
+ * - Initializing APM/OpenTelemetry
+ * - Transforming logging-related command-line flags into an environment variable
+ * so it doesn't interfere with Playwright's own command-line flags
+ */
+
+/**
+ * We disable node's version validation here as some IDEs will bundle their own
+ * Node.js version for extensions, or there's no way to get it to use Kibana's
+ * version, and it would cause the version validation check in setup_node_env to
+ * fail if the versions are out of sync.
+ */
+process.env.UNSAFE_DISABLE_NODE_VERSION_VALIDATION = 'true';
+require('../src/setup_node_env');

src/cli/apm.js

@@ -15,7 +15,7 @@ const { initTelemetry } = require('@kbn/telemetry');
 const rootDir = join(__dirname, '../..');
 const isKibanaDistributable = Boolean(build && build.distributable === true);
 
-module.exports = function (serviceName = name) {
-  initApm(process.argv, rootDir, isKibanaDistributable, serviceName);
-  initTelemetry(process.argv, rootDir, isKibanaDistributable, serviceName);
+module.exports = function (serviceName = name, argv = process.argv) {
+  initApm(argv, rootDir, isKibanaDistributable, serviceName);
+  initTelemetry(argv, rootDir, isKibanaDistributable, serviceName);
 };

src/core/packages/usage-data/server-internal/src/core_usage_data_service.test.ts

@@ -11,7 +11,6 @@ import type { ConfigPath } from '@kbn/config';
 import { BehaviorSubject, Observable } from 'rxjs';
 import { TestScheduler } from 'rxjs/testing';
 
-// eslint-disable-next-line @kbn/imports/no_unresolvable_imports
 import { HotObservable } from 'rxjs/internal/testing/HotObservable';
 
 import { configServiceMock } from '@kbn/config-mocks';

src/dev/license_checker/config.ts

@@ -96,4 +96,5 @@ export const LICENSE_OVERRIDES = {
   'language-subtag-registry@0.3.21': ['CC-BY-4.0'], // retired ODC‑By license https://github.com/mattcg/language-subtag-registry
   'buffers@0.1.1': ['MIT'], // license in importing module https://www.npmjs.com/package/binary
   '@bufbuild/protobuf@2.5.2': ['Apache-2.0'], // license (Apache-2.0 AND BSD-3-Clause)
+  '@arizeai/phoenix-client@2.3.4': ['Elastic License 2.0'], // see https://github.com/Arize-ai/phoenix/blob/main/LICENSE
 };