[ES|QL] Validation tests agains Elasticsearch

src/platform/packages/shared/kbn-esql-language/jest.integration.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration_node',
+  rootDir: '../../../../..',
+  roots: ['<rootDir>/src/platform/packages/shared/kbn-esql-language'],
+};

src/platform/packages/shared/kbn-esql-language/moon.yml

@@ -24,6 +24,8 @@ dependsOn:
   - '@kbn/core-pricing-common'
   - '@kbn/licensing-types'
   - '@kbn/field-types'
+  - '@kbn/test-es-server'
+  - '@kbn/tooling-log'
   - '@kbn/esql-scripts'
 tags:
   - shared-common
@@ -36,6 +38,7 @@ fileGroups:
   src:
     - src/**/*
     - '**/*.ts'
+    - jest.integration.config.js
     - '!target/**/*'
   jest-config:
     - jest.config.js

src/platform/packages/shared/kbn-esql-language/src/language/README.md

@@ -171,7 +171,7 @@ While AST requires the grammar to be compiled to be updated, definitions are sta
 Validation takes an AST as input and generates a list of messages to show to the user.
 The validation function leverages the definition files to check if the current AST is respecting the defined behaviour.
 Most of the logic rely purely on the definitions, but in some specific cases some ad-hoc conditions are defined within the code for specific commands/options.
-The validation test suite generates a set of fixtures at the end of its execution, which are then re-used for other test suites (i.e. some FTR integration tests) as `esql_validation_meta_tests.json`.
+The validation suites run as Jest unit tests and are also reused by Jest integration tests that compare client-side validation with a real Elasticsearch cluster.
 
 #### Autocomplete
 
@@ -201,21 +201,21 @@ The older pattern is
 
 - a single test file for each engine, one for validation, one for autocomplete. These were always large files and have only grown.
 - custom test methods: `testSuggestions` / `testErrorsAndWarnings`
-- validation cases are recorded in a JSON file which is then used to check our results against a live Elasticsearch instance in a functional test
+- validation cases were recorded in a JSON file and checked against Elasticsearch from FTR
 
 The newer pattern is
 
 - splitting the tests into multiple smaller files, all found in `__tests__` directories
 - standard test methods (`it`, `test`) with custom _assertion_ helpers
-- validation cases are checked against Elasticsearch by injecting assertion helpers run API integration tests. This does not require a JSON file.
+- validation suites are reused by Jest integration tests in the package. This does not require a JSON file.
 
 #### Validation
 
 ##### The new way
 
-Validation test logic is found in `src/platform/packages/shared/kbn-esql-language/src/validation/__tests__`.
+Validation test logic is found in `src/platform/packages/shared/kbn-esql-language/src/language/validation/__tests__`.
 
-Tests are found in files named with the following convention: `validation.some-description.test.ts`.
+Unit test wrappers live in `*.test.ts` files. Reusable suite logic lives in matching `*_suite.ts` files so the same tests can also run as Jest integration tests.
 
 Here is an example of a block in the new test format.
 
@@ -236,6 +236,10 @@ describe('TS <sources> [ <aggregates> [ BY <grouping> ]]', () => {
 
 `expectErrors` is created in the `setup()` factory. It has a very similar API to `testErrorsAndWarnings` however it is not itself a Jest test case. It is simply an assertion that is wrapped in a test case defined with the standard `test` or `it` function.
 
+The integration wrapper lives in `src/platform/packages/shared/kbn-esql-language/src/language/validation/integration_tests/validation_suites.test.ts`. It reuses the shared validation suites and checks client-side false positives against Elasticsearch. Run it with:
+
+`node scripts/jest_integration --config src/platform/packages/shared/kbn-esql-language/jest.integration.config.js`
+
 ##### The old way
 
 The old validation tests look like this
@@ -246,7 +250,7 @@ testErrorsAndWarnings(`ROW var = NOT 5 LIKE "?a"`, [
 ]);
 ```
 
-and are found in `src/platform/packages/shared/kbn-esql-language/src/validation/validation.test.ts`.
+and are found in `src/platform/packages/shared/kbn-esql-language/src/language/validation/validation.test.ts`.
 
 `testErrorsAndWarnings` supports `skip` and `only` modifiers e.g. `testErrorsAndWarnings.only('...')`.
 
@@ -256,8 +260,6 @@ It accepts
 2. a list of expected errors (can be empty)
 3. a list of expected warnings (can be empty or omitted)
 
-Running the tests in `validation.test.ts` populates `src/platform/packages/shared/kbn-esql-language/src/validation/esql_validation_meta_tests.json` which is then used in `src/platform/test/api_integration/apis/esql/errors.ts` to make sure our validator isn't giving users false positives. Therefore, the validation test suite should always be run after any changes have been made to it so that the JSON file stays in sync.
-
 #### Autocomplete
 
 ##### The new way
@@ -320,4 +322,4 @@ Its parameters are as follows
 1. the query
 2. the expected suggestions (can be strings or `Partial<ISuggestionItem>`)
 3. the trigger character. This should only be included if the test is intended to validate a "Trigger Character" trigger kind from Monaco ([ref](https://microsoft.github.io/monaco-editor/typedoc/enums/languages.CompletionTriggerKind.html#TriggerCharacter))
-4. custom callback data such as a list of indicies or a field list
+4. custom callback data such as a list of indicies or a field list

src/platform/packages/shared/kbn-esql-language/src/language/validation/__tests__/column_existence.test.ts

@@ -7,52 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { times } from 'lodash';
 import { setup } from './helpers';
+import { runColumnExistenceValidationSuite } from './column_existence_suite';
 
-describe('column existence checks', () => {
-  it('looks behind current command', async () => {
-    const { expectErrors } = await setup();
-    await expectErrors('FROM index | DROP keywordField | KEEP keywordField', [
-      'Unknown column "keywordField"',
-    ]);
-  });
-
-  it('treats FORK branches separately', async () => {
-    const { expectErrors } = await setup();
-    await expectErrors('FROM index | FORK (DROP keywordField) (KEEP keywordField)', []);
-  });
-
-  it('makes STATS generated columns available after inline WHERE filters', async () => {
-    const { expectErrors } = await setup();
-    await expectErrors(
-      'FROM index | STATS COUNT() WHERE integerField > 0 | EVAL result = `COUNT() WHERE integerField > 0` + 1',
-      []
-    );
-  });
-
-  it('returns a warning instead of an error when unmapped_fields is LOAD or NULLIFY', async () => {
-    const { expectErrors } = await setup();
-    await expectErrors(
-      'SET unmapped_fields = "LOAD"; FROM index | WHERE unmapped == ""',
-      [],
-      [
-        `"unmapped" column isn't mapped in any searched indices.\nIf you are not intentionally referencing an unmapped field,\ncheck that the field exists or that it is spelled correctly in your query.`,
-      ]
-    );
-  });
-
-  it('returns one warning for each instance of the same unmapped column', async () => {
-    const { expectErrors } = await setup();
-    await expectErrors(
-      'SET unmapped_fields = "LOAD"; FROM index | WHERE unmapped == "" | KEEP unmapped',
-      [],
-
-      times(
-        2,
-        () =>
-          `"unmapped" column isn't mapped in any searched indices.\nIf you are not intentionally referencing an unmapped field,\ncheck that the field exists or that it is spelled correctly in your query.`
-      )
-    );
-  });
-});
+runColumnExistenceValidationSuite(setup);

src/platform/packages/shared/kbn-esql-language/src/language/validation/__tests__/column_existence_suite.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { times } from 'lodash';
+import type { Setup } from './helpers';
+
+export const runColumnExistenceValidationSuite = (setup: Setup) => {
+  describe('column existence checks', () => {
+    it('looks behind current command', async () => {
+      const { expectErrors } = await setup();
+      await expectErrors('FROM index | DROP keywordField | KEEP keywordField', [
+        'Unknown column "keywordField"',
+      ]);
+    });
+
+    it('treats FORK branches separately', async () => {
+      const { expectErrors } = await setup();
+      await expectErrors('FROM index | FORK (DROP keywordField) (KEEP keywordField)', []);
+    });
+
+    it('makes STATS generated columns available after inline WHERE filters', async () => {
+      const { expectErrors } = await setup();
+      await expectErrors(
+        'FROM index | STATS COUNT() WHERE integerField > 0 | EVAL result = `COUNT() WHERE integerField > 0` + 1',
+        []
+      );
+    });
+
+    it('returns a warning instead of an error when unmapped_fields is LOAD or NULLIFY', async () => {
+      const { expectErrors } = await setup();
+      await expectErrors(
+        'SET unmapped_fields = "LOAD"; FROM index | WHERE unmapped == ""',
+        [],
+        [
+          `"unmapped" column isn't mapped in any searched indices.\nIf you are not intentionally referencing an unmapped field,\ncheck that the field exists or that it is spelled correctly in your query.`,
+        ]
+      );
+    });
+
+    it('returns one warning for each instance of the same unmapped column', async () => {
+      const { expectErrors } = await setup();
+      await expectErrors(
+        'SET unmapped_fields = "LOAD"; FROM index | WHERE unmapped == "" | KEEP unmapped',
+        [],
+
+        times(
+          2,
+          () =>
+            `"unmapped" column isn't mapped in any searched indices.\nIf you are not intentionally referencing an unmapped field,\ncheck that the field exists or that it is spelled correctly in your query.`
+        )
+      );
+    });
+  });
+};

src/platform/test/api_integration/apis/esql/index.ts → src/platform/packages/shared/kbn-esql-language/src/language/validation/__tests__/commands.test.ts

@@ -7,10 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import type { FtrProviderContext } from '../../ftr_provider_context';
+import { setup } from './helpers';
+import { runCommandsValidationSuite } from './commands_suite';
 
-export default function ({ loadTestFile }: FtrProviderContext) {
-  describe('ESQL sync', () => {
-    loadTestFile(require.resolve('./errors'));
-  });
-}
+runCommandsValidationSuite(setup);

src/platform/packages/shared/kbn-esql-language/src/language/validation/__tests__/commands_license.test.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { setup } from './helpers';
+import { runValidationCommandsLicenseSuite } from './commands_license_suite';
+
+runValidationCommandsLicenseSuite(setup);

src/platform/packages/shared/kbn-esql-language/src/language/validation/__tests__/commands_license_suite.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { Setup } from './helpers';
+
+export const runValidationCommandsLicenseSuite = (setup: Setup) => {
+  describe('Command license validation', () => {
+    it('Should allows licensed commands when user has required license', async () => {
+      const { expectErrors, callbacks } = await setup();
+
+      callbacks.getLicense = jest.fn(async () => ({
+        hasAtLeast: (license?: string) => license?.toLowerCase() === 'platinum',
+      }));
+
+      await expectErrors('FROM a_index | CHANGE_POINT doubleField', []);
+    });
+
+    it('should blocks licensed commands when user lacks required license', async () => {
+      const { expectErrors, callbacks } = await setup();
+
+      callbacks.getLicense = jest.fn(async () => ({
+        hasAtLeast: (license?: string) => license?.toLowerCase() !== 'platinum',
+      }));
+
+      await expectErrors('FROM a_index | CHANGE_POINT doubleField', [
+        'CHANGE_POINT requires a PLATINUM license.',
+      ]);
+    });
+  });
+};