diff --git a/app/vlselect/logsql/logsql.go b/app/vlselect/logsql/logsql.go index 220120c58f..93497ff2d4 100644 --- a/app/vlselect/logsql/logsql.go +++ b/app/vlselect/logsql/logsql.go @@ -50,9 +50,10 @@ var ( // { // "start":"YYYY-MM-DDThh:mm:sss.nnnnnnnnnZ", // "end":"YYYY-MM-DDThh:mm:sss.nnnnnnnnnZ", +// "hasTimeFilter":true|false // } func ProcessQueryTimeRangeRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) { - minTimestamp, maxTimestamp, err := parseQueryTimeRangeArgs(r) + minTimestamp, maxTimestamp, hasTimeFilter, err := parseQueryTimeRangeArgs(r) if err != nil { httpserver.Errorf(w, r, "%s", err) return @@ -62,25 +63,29 @@ func ProcessQueryTimeRangeRequest(ctx context.Context, w http.ResponseWriter, r startStr := timestampToRFC3339Nano(minTimestamp) endStr := timestampToRFC3339Nano(maxTimestamp) - fmt.Fprintf(w, `{"start":%q,"end":%q}`, startStr, endStr) + fmt.Fprintf(w, `{"start":%q,"end":%q,"hasTimeFilter":%t}`, startStr, endStr, hasTimeFilter) } -func parseQueryTimeRangeArgs(r *http.Request) (int64, int64, error) { +func parseQueryTimeRangeArgs(r *http.Request) (int64, int64, bool, error) { qStr := r.FormValue("query") if qStr == "" { - return 0, 0, fmt.Errorf("`query` arg cannot be empty") + return 0, 0, false, fmt.Errorf("`query` arg cannot be empty") } currTimestamp := time.Now().UnixNano() q, err := logstorage.ParseQueryAtTimestamp(qStr, currTimestamp) if err != nil { - return 0, 0, fmt.Errorf("cannot parse query [%s]: %s", qStr, err) + return 0, 0, false, fmt.Errorf("cannot parse query [%s]: %s", qStr, err) } minTimestamp, maxTimestamp := q.GetFilterTimeRange() + + // hasTimeFilter is true if the query itself contains a _time filter + hasTimeFilter := (minTimestamp != math.MinInt64 || maxTimestamp != math.MaxInt64) + if minTimestamp == math.MinInt64 { start, ok, err := getTimeNsec(r, "start") if err != nil { - return 0, 0, err + return 0, 0, false, err } if ok { minTimestamp = start @@ -89,14 +94,14 @@ func parseQueryTimeRangeArgs(r *http.Request) (int64, int64, error) { if maxTimestamp == math.MaxInt64 { end, ok, err := getTimeNsec(r, "end") if err != nil { - return 0, 0, err + return 0, 0, false, err } if ok { maxTimestamp = end } } - return minTimestamp, maxTimestamp, nil + return minTimestamp, maxTimestamp, hasTimeFilter, nil } func timestampToRFC3339Nano(nsec int64) string { diff --git a/app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx b/app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx index 5f88d5a44c..1435e949dd 100644 --- a/app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx +++ b/app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx @@ -10,6 +10,7 @@ import TimezonesPicker from "./Timezones/TimezonesPicker"; import ThemeControl from "../ThemeControl/ThemeControl"; import useDeviceDetect from "../../../hooks/useDeviceDetect"; import useBoolean from "../../../hooks/useBoolean"; +import QueryTimeOverride from "./QueryTimeOverride/QueryTimeOverride"; const title = "Settings"; @@ -48,6 +49,10 @@ const GlobalSettings = forwardRef((_, ref) => { show: true, component: }, + { + show: true, + component: + }, { show: !appModeEnable, component: diff --git a/app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/QueryTimeOverride/QueryTimeOverride.tsx b/app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/QueryTimeOverride/QueryTimeOverride.tsx new file mode 100644 index 0000000000..2c8b29711e --- /dev/null +++ b/app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/QueryTimeOverride/QueryTimeOverride.tsx @@ -0,0 +1,37 @@ +import { FC } from "preact/compat"; +import Switch from "../../../Main/Switch/Switch"; +import { getFromStorage, saveToStorage } from "../../../../utils/storage"; +import { useLocalStorageBoolean } from "../../../../hooks/useLocalStorageBoolean"; + +const key = "LOGS_OVERRIDE_TIME"; +const defaultValue = true; + +export const getOverrideValue = () => { + const value = getFromStorage(key); + if (!value) return defaultValue; + return value === "true"; +}; + +const QueryTimeOverride: FC = () => { + const [overrideTime, setOverrideTime] = useLocalStorageBoolean(key, getOverrideValue); + + const handleUpdateValue = (value: boolean) => { + setOverrideTime(value); + saveToStorage(key, String(value)); + }; + + return ( +
+
+ Query time override +
+ +
+ ); +}; + +export default QueryTimeOverride; diff --git a/app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/TimeSelector/TimeSelector.tsx b/app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/TimeSelector/TimeSelector.tsx index 04316c1915..94530c5bdd 100644 --- a/app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/TimeSelector/TimeSelector.tsx +++ b/app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/TimeSelector/TimeSelector.tsx @@ -18,6 +18,7 @@ import DateTimeInput from "../../../Main/DatePicker/DateTimeInput/DateTimeInput" import useBoolean from "../../../../hooks/useBoolean"; import useWindowSize from "../../../../hooks/useWindowSize"; import usePrevious from "../../../../hooks/usePrevious"; +import { useQueryState } from "../../../../state/query/QueryStateContext"; type Props = { onOpenSettings?: () => void; @@ -26,6 +27,8 @@ type Props = { export const TimeSelector: FC = ({ onOpenSettings }) => { const { isMobile } = useDeviceDetect(); const { isDarkTheme } = useAppState(); + const { queryHasTimeFilter } = useQueryState(); + const wrapperRef = useRef(null); const documentSize = useWindowSize(); const displayFullDate = useMemo(() => documentSize.width > 1120, [documentSize]); @@ -95,7 +98,7 @@ export const TimeSelector: FC = ({ onOpenSettings }) => { handleCloseOptions(); }; - const handleClickTimezone = () => { + const handleOpenSettings = () => { onOpenSettings && onOpenSettings(); handleCloseOptions(); }; @@ -164,6 +167,19 @@ export const TimeSelector: FC = ({ onOpenSettings }) => { })} ref={wrapperRef} > + {queryHasTimeFilter && ( +
+

Time range is overridden by the query `_time` filter.

+

Remove `_time` from the query to use manual selection.

+

+ To disable query time override in settings, click here. +

+
+ )} +
= ({ onOpenSettings }) => {
{activeTimezone.region}
{activeTimezone.utc}
diff --git a/app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/TimeSelector/style.scss b/app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/TimeSelector/style.scss index 1c71c75c89..e6205720e2 100644 --- a/app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/TimeSelector/style.scss +++ b/app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/TimeSelector/style.scss @@ -65,4 +65,27 @@ gap: $padding-small; } } + + &-warning { + display: flex; + flex-direction: column; + gap: 2px; + grid-column: 1/span 2; + color: $color-warning; + padding: 0 $padding-global $padding-global; + border-bottom: $border-divider; + margin-bottom: $padding-global; + line-height: 1.3; + font-size: $font-size-small; + + &__interactive { + cursor: pointer; + transition: filter 0.2s; + text-decoration: underline; + + &:hover { + filter: brightness(0.8); + } + } + } } diff --git a/app/vmui/packages/vmui/src/hooks/useLocalStorageBoolean.test.ts b/app/vmui/packages/vmui/src/hooks/useLocalStorageBoolean.test.ts index cab5928b47..04a3f2903d 100644 --- a/app/vmui/packages/vmui/src/hooks/useLocalStorageBoolean.test.ts +++ b/app/vmui/packages/vmui/src/hooks/useLocalStorageBoolean.test.ts @@ -67,4 +67,28 @@ describe("useLocalStorageBoolean", () => { expect(result.current[0]).toBe(false); }); + + it("uses customGetter instead of getFromStorage and reacts to storage events", () => { + const mockGetFromStorage = getFromStorage as Mock; + const customGetter = vi.fn(() => true); + + const { result } = renderHook(() => + useLocalStorageBoolean(testStorageKey, customGetter) + ); + + expect(customGetter).toHaveBeenCalledWith(testStorageKey); + expect(result.current[0]).toBe(true); + + expect(mockGetFromStorage).not.toHaveBeenCalled(); + + (customGetter as Mock).mockReturnValueOnce(false); + + act(() => { + window.dispatchEvent( + new StorageEvent("storage", { key: testStorageKey, newValue: "false" }) + ); + }); + + expect(result.current[0]).toBe(false); + }); }); diff --git a/app/vmui/packages/vmui/src/hooks/useLocalStorageBoolean.ts b/app/vmui/packages/vmui/src/hooks/useLocalStorageBoolean.ts index d0bba3aaae..6a677f7500 100644 --- a/app/vmui/packages/vmui/src/hooks/useLocalStorageBoolean.ts +++ b/app/vmui/packages/vmui/src/hooks/useLocalStorageBoolean.ts @@ -6,19 +6,27 @@ import useEventListener from "./useEventListener"; * A custom hook that synchronizes a boolean state with a value stored in localStorage. * * @param {StorageKeys} key - The key used to access the corresponding value in localStorage. + * @param {(key: StorageKeys) => boolean} [customGetter] - Optional custom getter function used to read * @returns {[boolean, function]} A tuple containing the current boolean value from localStorage and a setter function to update the value in localStorage. * * The hook listens to the "storage" event to automatically update the state when the localStorage value changes. */ -export const useLocalStorageBoolean = (key: StorageKeys): [boolean, (value: boolean) => void] => { - const [value, setValue] = useState(!!getFromStorage(key)); +export const useLocalStorageBoolean = ( + key: StorageKeys, + customGetter?: (key: StorageKeys) => boolean +): [boolean, (value: boolean) => void] => { + const getter = useCallback((key: StorageKeys) => { + return customGetter ? customGetter(key) : !!getFromStorage(key); + }, [key, customGetter]); + + const [value, setValue] = useState(getter(key)); const handleUpdateStorage = useCallback(() => { - const newValue = !!getFromStorage(key); + const newValue = getter(key); if (newValue !== value) { setValue(newValue); } - }, [key, value]); + }, [key, value, getter]); const setNewValue = useCallback((newValue: boolean) => { saveToStorage(key, newValue); diff --git a/app/vmui/packages/vmui/src/pages/QueryPage/QueryPage.tsx b/app/vmui/packages/vmui/src/pages/QueryPage/QueryPage.tsx index 8162bc276f..2f6ef27424 100644 --- a/app/vmui/packages/vmui/src/pages/QueryPage/QueryPage.tsx +++ b/app/vmui/packages/vmui/src/pages/QueryPage/QueryPage.tsx @@ -7,7 +7,7 @@ import Alert from "../../components/Main/Alert/Alert"; import QueryPageHeader from "./QueryPageHeader/QueryPageHeader"; import "./style.scss"; import { ErrorTypes, TimeParams } from "../../types"; -import { useTimeState } from "../../state/time/TimeStateContext"; +import { useTimeDispatch, useTimeState } from "../../state/time/TimeStateContext"; import { getFromStorage, saveToStorage } from "../../utils/storage"; import HitsChart from "./HitsChart/HitsChart"; import { useFetchLogHits } from "./hooks/useFetchLogHits"; @@ -23,6 +23,8 @@ import { ExtraFilter } from "../OverviewPage/FiltersBar/types"; import { useHitsChartConfig } from "./HitsChart/hooks/useHitsChartConfig"; import { useLimitGuard } from "./LimitController/useLimitGuard"; import LimitConfirmModal from "./LimitController/LimitConfirmModal"; +import { useFetchQueryTime } from "./hooks/useFetchQueryTime"; +import { getOverrideValue } from "../../components/Configurators/GlobalSettings/QueryTimeOverride/QueryTimeOverride"; const storageLimit = Number(getFromStorage("LOGS_LIMIT")); const defaultLimit = isNaN(storageLimit) ? LOGS_DEFAULT_LIMIT : storageLimit; @@ -30,9 +32,10 @@ const defaultLimit = isNaN(storageLimit) ? LOGS_DEFAULT_LIMIT : storageLimit; type FetchFlags = { logs: boolean; hits: boolean }; const QueryPage: FC = () => { - const { queryHistory } = useQueryState(); + const { queryHistory, queryHasTimeFilter } = useQueryState(); const queryDispatch = useQueryDispatch(); const { duration, relativeTime, period: periodState } = useTimeState(); + const timeDispatch = useTimeDispatch(); const { setSearchParamsFromKeys } = useSearchParamsFromObject(); const { topHits, groupFieldHits } = useHitsChartConfig(); const prevTopHits = usePrevious(topHits); @@ -49,6 +52,8 @@ const QueryPage: FC = () => { const [limit, setLimit] = useStateSearchParams(defaultLimit, LOGS_URL_PARAMS.LIMIT); const [query, setQuery] = useStateSearchParams("*", "query"); + const [skipNextPeriodEffect, setSkipNextPeriodEffect] = useState(false); + const handleChangeLimit = (limit: number) => { setLimit(limit); setSearchParamsFromKeys({ limit }); @@ -74,6 +79,7 @@ const QueryPage: FC = () => { const { logs, isLoading, error, fetchLogs, abortController, durationMs: queryDuration, queryParams } = useFetchLogs(query, limit); const { fetchLogHits, ...dataLogHits } = useFetchLogHits(query); + const { fetchQueryTime } = useFetchQueryTime(query); const fetchData = async (period: TimeParams, flags: FetchFlags) => { if (flags.logs) { @@ -95,14 +101,32 @@ const QueryPage: FC = () => { return getTimeperiodForDuration(duration, until()); }; - const handleRunQuery = () => { + const handleRunQuery = async () => { if (!query) { setQueryError(ErrorTypes.validQuery); return; } setQueryError(""); - const newPeriod = getPeriod(); + const uiPeriod = getPeriod(); + const apiPeriod = getOverrideValue() + ? await fetchQueryTime({ query, period: uiPeriod }) + : undefined; + + const newPeriod = apiPeriod ?? uiPeriod; + + queryDispatch({ type: "SET_QUERY_HAS_TIME_FILTER", payload: !!apiPeriod?.hasTimeFilter }); + if (apiPeriod?.hasTimeFilter) { + setSkipNextPeriodEffect(true); + timeDispatch({ + type: "SET_PERIOD", + payload: { + from: new Date(newPeriod.start * 1000), + to: new Date(newPeriod.end * 1000) + } + }); + } + setPeriod(newPeriod); dataLogHits.abortController.abort?.(); abortController.abort?.(); @@ -115,6 +139,7 @@ const QueryPage: FC = () => { }); updateHistory(); }; + const handleApplyFilter = (val: ExtraFilter) => { setQuery(prev => `${filterToExpr(val)} AND ${prev}`); setIsUpdatingQuery(true); @@ -131,6 +156,10 @@ const QueryPage: FC = () => { useEffect(() => { if (!query) return; + if (skipNextPeriodEffect) { + setSkipNextPeriodEffect(false); + return; + } handleRunQuery(); }, [periodState]); @@ -174,6 +203,13 @@ const QueryPage: FC = () => { isLoading={isLoading || dataLogHits.isLoading} /> {error && {error}} + {queryHasTimeFilter && +

+ Time range is overridden by the query `_time` filter. + Remove `_time` from the query to use manual selection. + Disable query time override in Settings. +

+ } {!error && ( { + const { serverUrl } = useAppState(); + const tenant = useTenant(); + + const [serverPeriod, setServerPeriod] = useState(null); + const [isLoading, setLoading] = useState(false); + const [error, setError] = useState(); + + const fetchQueryTime = useCallback(async ({ period, query }: { + period: TimeParams, + query?: string + }): Promise => { + const params = new URLSearchParams({ + query: query || defaultQuery || "", + start: period.start.toString(), + end: period.end.toString(), + }); + + setServerPeriod(null); + setLoading(true); + setError(""); + + try { + const url = `${serverUrl}/select/logsql/query_time_range`; + const response = await fetch(url, { + method: "POST", + headers: { ...tenant }, + body: params, + }); + + if (!response.ok || !response.body) { + const text = await response.text(); + setError(text); + setLoading(false); + return; + } + + const { start, end, hasTimeFilter }: ResponseTimeRange = await response.json(); + const startDate = dayjs(start); + const endDate = dayjs(end); + + if (!startDate.isValid() || !endDate.isValid()) { + const text = "Invalid date range"; + setError(text); + setLoading(false); + return; + } + + const timeRange = { from: startDate.toDate(), to: endDate.toDate() }; + const durationPeriod = getDurationFromPeriod(timeRange); + const period = getTimeperiodForDuration(durationPeriod, timeRange.to); + const serverPeriod = { ...period, hasTimeFilter }; + setServerPeriod(serverPeriod); + return serverPeriod; + } catch (e) { + if (e instanceof Error && e.name !== "AbortError") { + setError(String(e)); + console.error(e); + } + } finally { + setLoading(false); + } + }, [defaultQuery, serverUrl, tenant]); + + + return { + fetchQueryTime, + serverPeriod, + isLoading, + error, + }; +}; diff --git a/app/vmui/packages/vmui/src/state/query/reducer.ts b/app/vmui/packages/vmui/src/state/query/reducer.ts index 45190b2ac1..87112240b7 100644 --- a/app/vmui/packages/vmui/src/state/query/reducer.ts +++ b/app/vmui/packages/vmui/src/state/query/reducer.ts @@ -6,6 +6,7 @@ import { QueryAutocompleteCacheItem } from "../../components/Configurators/QueryEditor/QueryAutocompleteCache"; import { AutocompleteOptions } from "../../components/Main/Autocomplete/Autocomplete"; +import { getOverrideValue } from "../../components/Configurators/GlobalSettings/QueryTimeOverride/QueryTimeOverride"; export interface QueryHistoryType { index: number; @@ -19,6 +20,7 @@ export interface QueryState { autocompleteQuick: boolean; autocompleteCache: QueryAutocompleteCache; metricsQLFunctions: AutocompleteOptions[]; + queryHasTimeFilter: boolean; } export type QueryAction = @@ -28,6 +30,7 @@ export type QueryAction = | { type: "TOGGLE_AUTOCOMPLETE" } | { type: "SET_AUTOCOMPLETE_QUICK", payload: boolean } | { type: "SET_AUTOCOMPLETE_CACHE", payload: { key: QueryAutocompleteCacheItem, value: string[] } } + | { type: "SET_QUERY_HAS_TIME_FILTER", payload: boolean } const query = getQueryArray(); export const initialQueryState: QueryState = { @@ -37,6 +40,7 @@ export const initialQueryState: QueryState = { autocompleteQuick: false, autocompleteCache: new QueryAutocompleteCache(), metricsQLFunctions: [], + queryHasTimeFilter: false, }; export function reducer(state: QueryState, action: QueryAction): QueryState { @@ -75,6 +79,11 @@ export function reducer(state: QueryState, action: QueryAction): QueryState { ...state }; } + case "SET_QUERY_HAS_TIME_FILTER": + return { + ...state, + queryHasTimeFilter: getOverrideValue() ? action.payload : false + }; default: throw new Error(); } diff --git a/app/vmui/packages/vmui/src/utils/storage.ts b/app/vmui/packages/vmui/src/utils/storage.ts index 0b92163222..3cf5e63704 100644 --- a/app/vmui/packages/vmui/src/utils/storage.ts +++ b/app/vmui/packages/vmui/src/utils/storage.ts @@ -21,6 +21,7 @@ export type StorageKeys = "AUTOCOMPLETE" | "METRICS_QUERY_HISTORY" | "SERVER_URL" | "RAW_JSON_LIVE_VIEW" + | "LOGS_OVERRIDE_TIME" | DeprecatedStorageKeys; diff --git a/docs/victorialogs/CHANGELOG.md b/docs/victorialogs/CHANGELOG.md index 224ad79ca1..e7d4fbdb56 100644 --- a/docs/victorialogs/CHANGELOG.md +++ b/docs/victorialogs/CHANGELOG.md @@ -21,8 +21,10 @@ according to the follosing docs: ## tip + * FEATURE: [syslog data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/): add support for automatic parsing of [`@cee` messages](http://cee.mitre.org/language/1.0-beta1/clt.html#syslog). Thanks to @exherb for the pull request [#842](https://github.com/VictoriaMetrics/VictoriaLogs/pull/842). * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add the help button with shortcuts reference and controls for charts and query input. See [#77](https://github.com/VictoriaMetrics/VictoriaLogs/issues/77). +* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): auto-sync time picker with `_time` filter from the query (can be disabled). See [#558](https://github.com/VictoriaMetrics/VictoriaLogs/issues/558). * BUGFIX: [delete API](https://docs.victoriametrics.com/victorialogs/#how-to-delete-logs): prevent from possible fatal error (panic) at `block_stream_merger.go:237` during the deletion of the logs. See [#825](https://github.com/VictoriaMetrics/VictoriaLogs/issues/825).