[ES|QL] Supports timezone (elastic#247917)

## Summary

Closes elastic#172221

ES|QL now supports timezones! This PR is adding support to:

- our esql expression. The applications that are using the expression
are getting this for free
- consumers of the getESQLResults utility
- all analyst xp apps

There are usages (such as in ML) that are still not supporting it. The
teams need to handle it themselves. I didnt do it mostly for 2 reasons:

- I dont know the apps and the code
- it is very complicated to do so

### When we will merge
I am going to merge it after the next serverless release (next week) to
be sure that all the ES serverless instances have the new request
parameter already.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

Author: stratoula

src/platform/packages/private/kbn-generate-csv/src/generate_csv_esql.test.ts

@@ -317,7 +317,7 @@ describe('CsvESQLGenerator', () => {
       );
 
       expect(mockDataClientSearchFn).toBeCalledWith(
-        { params: { filter: undefined, locale: 'en', query: '' } },
+        { params: { filter: undefined, locale: 'en', query: '', time_zone: 'America/New_York' } },
         {
           strategy: 'esql',
           transport: {
@@ -396,7 +396,7 @@ describe('CsvESQLGenerator', () => {
       );
 
       expect(mockDataClientSearchFn).toBeCalledWith(
-        { params: { filter: undefined, locale: 'en', query: '' } },
+        { params: { filter: undefined, locale: 'en', query: '', time_zone: 'America/New_York' } },
         {
           strategy: 'esql',
           transport: {
@@ -494,6 +494,7 @@ describe('CsvESQLGenerator', () => {
             },
             locale: 'en',
             query: 'FROM kibana_sample_data_logs | LIMIT 10 | LIMIT 500',
+            time_zone: 'America/New_York',
           },
         },
         {
@@ -568,6 +569,7 @@ describe('CsvESQLGenerator', () => {
             ]),
             locale: 'en',
             query: `${query.esql} | LIMIT 500`,
+            time_zone: 'America/New_York',
           },
         },
         {
@@ -607,6 +609,7 @@ describe('CsvESQLGenerator', () => {
             filter: undefined,
             locale: 'en',
             query: 'FROM custom-metrics | LIMIT 500',
+            time_zone: 'America/New_York',
           },
         },
         {

src/platform/packages/private/kbn-generate-csv/src/generate_csv_esql.ts

@@ -74,7 +74,7 @@ export class CsvESQLGenerator {
     let reportingError: undefined | ReportingError;
     const warnings: string[] = [];
 
-    const { maxSizeBytes, maxRows, bom, escapeFormulaValues } = settings;
+    const { maxSizeBytes, maxRows, bom, escapeFormulaValues, timezone } = settings;
     const builder = new MaxSizeStringBuilder(this.stream, byteSizeValueToNumber(maxSizeBytes), bom);
 
     // it will return undefined if there are no _tstart, _tend named params in the query
@@ -130,10 +130,7 @@ export class CsvESQLGenerator {
         // locale can be used for number/date formatting
         locale: i18n.getLocale(),
         ...(params.length ? { params } : {}),
-        // TODO: time_zone support was temporarily removed from ES|QL,
-        // we will need to add it back in once it is supported again.
-        // https://github.com/elastic/elasticsearch/pull/102767
-        // timezone
+        time_zone: timezone,
       },
     };
 

src/platform/packages/shared/kbn-es-query/index.ts

@@ -143,6 +143,7 @@ export {
   isDataViewFieldSubtypeMulti,
   isDataViewFieldSubtypeNested,
   isCCSRemoteIndexName,
+  getTimeZoneFromSettings,
 } from './src/utils';
 
 export type { ExecutionContextSearch } from './src/expressions/types';

src/platform/packages/shared/kbn-es-types/src/search.ts

@@ -675,10 +675,7 @@ export interface ESQLSearchResponse {
 }
 
 export interface ESQLSearchParams {
-  // TODO: time_zone support was temporarily removed from ES|QL,
-  // we will need to add it back in once it is supported again.
-  // https://github.com/elastic/elasticsearch/pull/102767
-  // time_zone?: string;
+  time_zone?: string;
   query: string;
   filter?: unknown;
   project_routing?: string;

src/platform/packages/shared/kbn-esql-utils/src/utils/run_query.ts

@@ -12,6 +12,7 @@ import dateMath from '@kbn/datemath';
 import type { DatatableColumn } from '@kbn/expressions-plugin/common';
 import type { ISearchGeneric } from '@kbn/search-types';
 import type { TimeRange } from '@kbn/es-query';
+import { getTimeZoneFromSettings } from '@kbn/es-query';
 import { esFieldTypeToKibanaFieldType } from '@kbn/field-types';
 import type { ESQLColumn, ESQLSearchResponse, ESQLSearchParams } from '@kbn/es-types';
 import { lastValueFrom } from 'rxjs';
@@ -177,6 +178,7 @@ export async function getESQLResults({
   dropNullColumns,
   timeRange,
   variables,
+  timezone,
 }: {
   esqlQuery: string;
   search: ISearchGeneric;
@@ -185,6 +187,7 @@ export async function getESQLResults({
   dropNullColumns?: boolean;
   timeRange?: TimeRange;
   variables?: ESQLControlVariable[];
+  timezone?: string;
 }): Promise<{
   response: ESQLSearchResponse;
   params: ESQLSearchParams;
@@ -198,6 +201,7 @@ export async function getESQLResults({
           query: esqlQuery,
           ...(dropNullColumns ? { dropNullColumns: true } : {}),
           ...(namedParams.length ? { params: namedParams } : {}),
+          ...(timezone ? { time_zone: getTimeZoneFromSettings(timezone) } : {}),
         },
       },
       {

src/platform/plugins/shared/controls/public/controls/esql_control/utils/get_esql_single_column_values.ts

@@ -10,7 +10,9 @@
 import type { ISearchGeneric } from '@kbn/search-types';
 import type { TimeRange } from '@kbn/es-query';
 import { getESQLResults } from '@kbn/esql-utils';
+import { UI_SETTINGS } from '@kbn/data-plugin/public';
 import type { ESQLControlVariable } from '@kbn/esql-types';
+import { coreServices } from '../../../services/kibana_services';
 
 export interface GetESQLSingleColumnValuesSuccess {
   values: string[];
@@ -35,13 +37,15 @@ export const getESQLSingleColumnValues = async ({
   GetESQLSingleColumnValuesSuccess | GetESQLSingleColumnValuesFailure
 > => {
   try {
+    const timezone = coreServices.uiSettings?.get<'Browser' | string>(UI_SETTINGS.DATEFORMAT_TZ);
     const results = await getESQLResults({
       esqlQuery: query,
       search,
       signal: undefined,
       filter: undefined,
       dropNullColumns: true,
       timeRange,
+      timezone,
       variables: esqlVariables,
     });
     const columns = results.response.columns.map((col) => col.name);

src/platform/plugins/shared/data/common/search/expressions/esql.test.ts

@@ -28,7 +28,12 @@ describe('getEsqlFn', () => {
     const esqlFn = getEsqlFn({
       getStartDependencies: async () => ({
         search: mockSearch,
-        uiSettings: {} as UiSettingsCommon,
+        uiSettings: {
+          get: jest.fn((key: string) => {
+            if (key === 'dateFormat:tz') return 'UTC';
+            return undefined;
+          }),
+        } as unknown as UiSettingsCommon,
       }),
     });
 

src/platform/plugins/shared/data/common/search/expressions/esql.ts

@@ -32,7 +32,7 @@ import {
 import { zipObject } from 'lodash';
 import type { Observable } from 'rxjs';
 import { catchError, defer, map, switchMap, tap, throwError } from 'rxjs';
-import { buildEsQuery, type Filter } from '@kbn/es-query';
+import { buildEsQuery, type Filter, getTimeZoneFromSettings } from '@kbn/es-query';
 import type { ESQLSearchParams, ESQLSearchResponse } from '@kbn/es-types';
 import DateMath from '@kbn/datemath';
 import { getEsQueryConfig } from '../../es_query';
@@ -60,10 +60,6 @@ type Output = Observable<Datatable>;
 
 interface Arguments {
   query: string;
-  // TODO: time_zone support was temporarily removed from ES|QL,
-  // we will need to add it back in once it is supported again.
-  // https://github.com/elastic/elasticsearch/pull/102767
-  // timezone?: string;
   timeField?: string;
   locale?: string;
 
@@ -117,15 +113,6 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
           defaultMessage: 'An ES|QL query.',
         }),
       },
-      // timezone: {
-      //   aliases: ['tz'],
-      //   types: ['string'],
-      //   default: 'UTC',
-      //   help: i18n.translate('data.search.esql.timezone.help', {
-      //     defaultMessage:
-      //       'The timezone to use for date operations. Valid ISO8601 formats and UTC offsets both work.',
-      //   }),
-      // },
       timeField: {
         aliases: ['timeField'],
         types: ['string'],
@@ -169,14 +156,7 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
     },
     fn(
       input,
-      {
-        query,
-        /* timezone, */ timeField,
-        locale,
-        titleForInspector,
-        descriptionForInspector,
-        ignoreGlobalFilters,
-      },
+      { query, timeField, locale, titleForInspector, descriptionForInspector, ignoreGlobalFilters },
       { abortSignal, inspectorAdapters, getKibanaRequest, getSearchSessionId }
     ) {
       return defer(() =>
@@ -197,17 +177,18 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
           // and the query is not set with ?? in the query, we should set it
           // https://github.com/elastic/elasticsearch/pull/122459
           const fixedQuery = fixESQLQueryWithVariables(query, input?.esqlVariables ?? []);
+          const esQueryConfigs = getEsQueryConfig(
+            uiSettings as Parameters<typeof getEsQueryConfig>[0]
+          );
           const params: ESQLSearchParams = {
             query: fixedQuery,
-            // time_zone: timezone,
+            time_zone: esQueryConfigs.dateFormatTZ
+              ? getTimeZoneFromSettings(esQueryConfigs.dateFormatTZ)
+              : 'UTC',
             locale,
             include_execution_metadata: true,
           };
           if (input) {
-            const esQueryConfigs = getEsQueryConfig(
-              uiSettings as Parameters<typeof getEsQueryConfig>[0]
-            );
-
             const namedParams = getNamedParams(fixedQuery, input.timeRange, input.esqlVariables);
 
             if (namedParams.length) {

src/platform/plugins/shared/esql/public/triggers/esql_controls/control_flyout/index.tsx

@@ -220,7 +220,6 @@ export function ESQLControlsFlyout({
         search={search}
         valuesRetrieval={valuesField}
         timeRange={timeRange}
-        currentApp={currentApp}
         esqlVariables={esqlVariables}
       />
     ) : (

src/platform/plugins/shared/esql/public/triggers/esql_controls/control_flyout/value_control_form.tsx

@@ -30,6 +30,7 @@ import type { ISearchGeneric } from '@kbn/search-types';
 import { isEqual } from 'lodash';
 import React, { useCallback, useEffect, useState } from 'react';
 import useMountedState from 'react-use/lib/useMountedState';
+import { UI_SETTINGS } from '@kbn/data-plugin/public';
 import { ESQLLangEditor } from '../../../create_editor';
 import type { ServiceDeps } from '../../../kibana_services';
 import { ChooseColumnPopover } from './choose_column_popover';
@@ -45,7 +46,6 @@ interface ValueControlFormProps {
   initialState?: ESQLControlState;
   valuesRetrieval?: string;
   timeRange?: TimeRange;
-  currentApp?: string;
   esqlVariables: ESQLControlVariable[];
 }
 
@@ -67,7 +67,6 @@ export function ValueControlForm({
   setControlState,
   valuesRetrieval,
   timeRange,
-  currentApp,
   esqlVariables,
 }: ValueControlFormProps) {
   const isMounted = useMountedState();
@@ -158,13 +157,15 @@ export function ValueControlForm({
   const onValuesQuerySubmit = useCallback(
     async (query: string) => {
       try {
+        const timezone = core.uiSettings.get<'Browser' | string>(UI_SETTINGS.DATEFORMAT_TZ);
         getESQLResults({
           esqlQuery: query,
           search,
           signal: undefined,
           filter: undefined,
           dropNullColumns: true,
           timeRange,
+          timezone,
           variables: esqlVariables,
         }).then((results) => {
           if (!isMounted()) {
@@ -195,7 +196,7 @@ export function ValueControlForm({
         setEsqlQueryErrors([e]);
       }
     },
-    [isMounted, search, timeRange, esqlVariables]
+    [isMounted, search, timeRange, esqlVariables, core.uiSettings]
   );
 
   const setSuggestedQuery = useCallback(async () => {