diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ae3db91..82eb9032 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * FEATURE: fetch tenants from the VictoriaLogs backend and allow selecting a tenant in the datasource settings. See [#475](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/475). * FEATURE: enable client side caching and make reliable behavior in QueryBuilder filters. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/357). * FEATURE: add compatibility with Grafana 10.x and 11.x by using dynamic component loading for Combobox. +* FEATURE: add quick level filter, which allows filtering logs by level according to `Log Level Rules` and the base level field. It is the first part of the [issue #108](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/108). See [pr #495](https://github.com/VictoriaMetrics/victorialogs-datasource/pull/495). ## v0.22.4 diff --git a/package.json b/package.json index 5e9a4917..2fa9a47e 100644 --- a/package.json +++ b/package.json @@ -34,9 +34,10 @@ "@testing-library/jest-dom": "6.4.6", "@testing-library/react": "16.0.0", "@types/jest": "^29.5.14", - "@types/lodash": "^4.17.5", + "@types/lodash": "^4.17.21", "@types/node": "^20.14.9", "@types/react": "^19.2.2", + "@types/semver": "^7.7.0", "@types/testing-library__jest-dom": "6.0.0", "@typescript-eslint/eslint-plugin": "^8.26.1", "@typescript-eslint/parser": "^8.26.1", @@ -88,7 +89,7 @@ "@grafana/ui": "12.2.0", "@lezer/common": "^1.2.1", "@lezer/lr": "^1.4.1", - "@types/semver": "^7.7.0", + "lodash": "^4.17.21", "react": "18.3.1", "react-dom": "18.3.1", "rxjs": "^7.8.2", diff --git a/src/components/QueryEditor/EditorHeader.tsx b/src/components/QueryEditor/EditorHeader.tsx index f8d26404..97c2420e 100644 --- a/src/components/QueryEditor/EditorHeader.tsx +++ b/src/components/QueryEditor/EditorHeader.tsx @@ -19,7 +19,7 @@ const getStyles = (theme: GrafanaTheme2) => ({ display: 'flex', flexWrap: 'wrap', alignItems: 'center', - justifyContent: 'flex-end', + justifyContent: 'space-between', gap: theme.spacing(1), minHeight: theme.spacing(4), }), diff --git a/src/components/QueryEditor/QueryEditor.tsx b/src/components/QueryEditor/QueryEditor.tsx index 516b1dbb..6a5ba8e3 100644 --- a/src/components/QueryEditor/QueryEditor.tsx +++ b/src/components/QueryEditor/QueryEditor.tsx @@ -3,10 +3,11 @@ import { isEqual } from 'lodash'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { CoreApp, GrafanaTheme2, LoadingState } from '@grafana/data'; -import { Button, ConfirmModal, useStyles2 } from '@grafana/ui'; +import { Button, ConfirmModal, Stack, useStyles2 } from '@grafana/ui'; import { getQueryExprVariableRegExp } from "../../LogsQL/regExpOperator"; import { isExprHasStatsPipeFunctions } from "../../LogsQL/statsPipeFunctions"; +import { LevelQueryFilter } from "../../configuration/LogLevelRules/LevelQueryFilter/LeveQueryFilter"; import { storeKeys } from "../../store/constants"; import store from "../../store/store"; import { Query, QueryEditorMode, QueryType, VictoriaLogsQueryEditorProps } from "../../types"; @@ -90,25 +91,29 @@ const QueryEditor = React.memo((props) => { />
- {showStatsWarn && ()} - - - - {app !== CoreApp.Explore && app !== CoreApp.Correlations && ( - - )} + {app === CoreApp.Explore && + } + + {showStatsWarn && ()} + + + + {app !== CoreApp.Explore && app !== CoreApp.Correlations && ( + + )} +
{editorMode === QueryEditorMode.Builder ? ( diff --git a/src/components/QueryEditor/QueryHints/hints/types.ts b/src/components/QueryEditor/QueryHints/hints/types.ts new file mode 100644 index 00000000..00771f53 --- /dev/null +++ b/src/components/QueryEditor/QueryHints/hints/types.ts @@ -0,0 +1,14 @@ +export interface QueryHints { + sections: QueryHintSection[]; +} + +export interface QueryHintSection { + title: string; + hints: QueryHint[]; +} + +export interface QueryHint { + title: string; + queryExpr: string; + description?: string; +} diff --git a/src/components/QueryEditor/QueryHints/hints/useLevelQueryHintSection.ts b/src/components/QueryEditor/QueryHints/hints/useLevelQueryHintSection.ts new file mode 100644 index 00000000..dd6b4518 --- /dev/null +++ b/src/components/QueryEditor/QueryHints/hints/useLevelQueryHintSection.ts @@ -0,0 +1,56 @@ +import { groupBy } from 'lodash'; +import { useMemo } from "react"; + +import { LogLevel } from "@grafana/data"; + +import { + OperatorLabels, + possibleLogValueByLevelType, + UNIQ_LOG_LEVEL, + UniqLogLevelKeys +} from "../../../../configuration/LogLevelRules/const"; +import { LogLevelRule } from "../../../../configuration/LogLevelRules/types"; + +import { QueryHint, QueryHintSection } from "./types"; + +export const useLevelQueryHintSection = (levelRules: LogLevelRule[]): QueryHintSection => { + return useMemo(() => { + const enabledLevelRules = levelRules.filter(rule => rule.enabled); + const groupedByLevelRules = groupBy(enabledLevelRules, 'level'); + const levelFilters = Object + .values(UNIQ_LOG_LEVEL) + .filter(val => val !== LogLevel.unknown) + .reduce((acc, logLevel) => { + acc[logLevel] = groupedByLevelRules[logLevel] || []; + return acc; + }, {} as Record); + + const hints = Object + .entries(levelFilters) + .map(([ruleLevel, rule]): QueryHint => { + const levelKey = ruleLevel as UniqLogLevelKeys; + const queryExprByRules = rule.map(r => `${r.field}:${OperatorLabels[r.operator]}"${r.value}"`); + const possibleLevelValues = possibleLogValueByLevelType[levelKey].map(value => `"${value}"`).join(','); + const queryExprByLevel = `level:contains_common_case(${possibleLevelValues})`; + const queryParts = [queryExprByLevel]; + if (queryExprByRules.length > 0) { + queryParts.push(...queryExprByRules); + } + const expr = queryParts.join(' OR '); + return { + title: levelKey, + queryExpr: expr + }; + }); + + hints.push({ + title: LogLevel.unknown, + queryExpr: `!(${hints.map(hint => hint.queryExpr).join(' OR ')})` + }); + + return { + title: 'Filter by log level', + hints + }; + }, [levelRules]); +}; diff --git a/src/configuration/LogLevelRules/LevelQueryFilter/LeveQueryFilter.tsx b/src/configuration/LogLevelRules/LevelQueryFilter/LeveQueryFilter.tsx new file mode 100644 index 00000000..0bc05a30 --- /dev/null +++ b/src/configuration/LogLevelRules/LevelQueryFilter/LeveQueryFilter.tsx @@ -0,0 +1,58 @@ +import React, { MouseEvent, useCallback, useMemo } from "react"; + + +import { LogLevel } from "@grafana/data"; +import { Stack } from "@grafana/ui"; + +import { useLevelQueryHintSection } from "../../../components/QueryEditor/QueryHints/hints/useLevelQueryHintSection"; +import { Query } from "../../../types"; +import { LogLevelRule } from "../types"; + +import { LevelFilterButton } from "./LevelFilterButton"; +import { buildQueryExprWithLevelFilters } from "./utils"; + + +interface Props { + logLevelRules: LogLevelRule[]; + query: Query; + onChange: (value: Query) => void; +} + +export const LevelQueryFilter = ({ logLevelRules, query, onChange }: Props) => { + const levelQueryHintSection = useLevelQueryHintSection(logLevelRules); + + const unknownLevelFilter = useMemo(() => levelQueryHintSection.hints.find(hint => hint.title === LogLevel.unknown), [levelQueryHintSection]); + const isQueryContainUnknowFilter = unknownLevelFilter && query.expr.includes(unknownLevelFilter.queryExpr); + + const handleClick = useCallback((e: MouseEvent, levelQueryExpr: string, title: string) => { + const isShiftPressed = e.shiftKey; + const isUnknownFilter = title === LogLevel.unknown; + const queryExpr = buildQueryExprWithLevelFilters({ + queryExpr: query.expr, + levelQueryExpr, + isShiftPressed, + isQueryContainUnknowFilter, + isUnknownFilter + }) + onChange({ ...query, expr: queryExpr }); + }, [isQueryContainUnknowFilter, onChange, query]); + + return ( + + {levelQueryHintSection.hints.map(({ title, queryExpr }) => { + const isNegativeStart = query.expr.startsWith('!('); + const isSelected = (isNegativeStart && title === LogLevel.unknown) + || (!isNegativeStart && query.expr.includes(queryExpr)); + return ( + ) => handleClick(e, queryExpr, title)} + level={title as LogLevel} + label={title} + isSelected={isSelected} + /> + ) + })} + + ); +}; diff --git a/src/configuration/LogLevelRules/LevelQueryFilter/LevelFilterButton.tsx b/src/configuration/LogLevelRules/LevelQueryFilter/LevelFilterButton.tsx new file mode 100644 index 00000000..e02a9df7 --- /dev/null +++ b/src/configuration/LogLevelRules/LevelQueryFilter/LevelFilterButton.tsx @@ -0,0 +1,48 @@ +import { css } from "@emotion/css"; +import React, { MouseEvent } from "react"; + +import { LogLevel } from "@grafana/data"; +import { Button, Stack, useStyles2 } from "@grafana/ui"; + +import { LOG_LEVEL_COLOR } from "../const"; + +export interface LevelFilterButtonProps { + onClick: (e: MouseEvent) => void; + label: string; + level: LogLevel; + isSelected?: boolean; +} + +export const LevelFilterButton = ({ onClick, label, level, isSelected }: LevelFilterButtonProps) => { + const styles = useStyles2(getStyles); + return ( + + + + ); +}; + +const getLogLevelColor = (level: LogLevel): string => { + return LOG_LEVEL_COLOR[level] || LOG_LEVEL_COLOR[LogLevel.unknown]; +}; + +const getStyles = () => { + return { + colorCircle: css({ + width: 12, + height: 12, + borderRadius: '50%', + marginRight: 5, + }), + }; +}; diff --git a/src/configuration/LogLevelRules/LevelQueryFilter/utils.test.ts b/src/configuration/LogLevelRules/LevelQueryFilter/utils.test.ts new file mode 100644 index 00000000..83aaa1f0 --- /dev/null +++ b/src/configuration/LogLevelRules/LevelQueryFilter/utils.test.ts @@ -0,0 +1,74 @@ +import { buildQueryExprWithLevelFilters } from './utils'; + +describe('buildQueryExprWithLevelFilters', () => { + it('should return levelQueryExpr if queryExpr is empty', () => { + const result = buildQueryExprWithLevelFilters({ + queryExpr: '', + levelQueryExpr: 'level: info', + isUnknownFilter: false, + isShiftPressed: false, + }); + expect(result).toBe('level: info'); + }); + + it('should handle queries without pipes and add the level query', () => { + const result = buildQueryExprWithLevelFilters({ + queryExpr: 'filterName1: filterValue1', + levelQueryExpr: 'level: error', + isUnknownFilter: false, + isShiftPressed: false, + }); + expect(result).toBe('level: error | filterName1: filterValue1'); + }); + + it('should prepend the level query if a level filter already exists and shift is pressed', () => { + const result = buildQueryExprWithLevelFilters({ + queryExpr: 'filterName1: filterValue1 level: warning', + levelQueryExpr: 'level: error', + isUnknownFilter: false, + isShiftPressed: true, + }); + expect(result).toBe('level: error OR filterName1: filterValue1 level: warning'); + }); + + it('should replace the whole query if a level filter already exists and shift is not pressed', () => { + const result = buildQueryExprWithLevelFilters({ + queryExpr: 'filterName1: filterValue1 level: warning', + levelQueryExpr: 'level: error', + isUnknownFilter: false, + isShiftPressed: false, + }); + expect(result).toBe('level: error'); + }); + + it('should append the level query when multiple filters are piped and no level filter exists', () => { + const result = buildQueryExprWithLevelFilters({ + queryExpr: 'filterName1: filterValue1 | filterName2: filterValue2', + levelQueryExpr: 'level: debug', + isUnknownFilter: false, + isShiftPressed: false, + }); + expect(result).toBe('level: debug | filterName1: filterValue1 | filterName2: filterValue2'); + }); + + it('should handle unknown filters appropriately if isUnknownFilter is true', () => { + const result = buildQueryExprWithLevelFilters({ + queryExpr: 'filterName1: filterValue1', + levelQueryExpr: 'level: unknown', + isUnknownFilter: true, + isShiftPressed: false, + }); + expect(result).toBe('level: unknown | filterName1: filterValue1'); + }); + + + it('should replace current level filter by unknown filter even if the shift is pressed', () => { + const result = buildQueryExprWithLevelFilters({ + queryExpr: 'level: info', + levelQueryExpr: 'level: unknown', + isUnknownFilter: true, + isShiftPressed: true, + }); + expect(result).toBe('level: unknown'); + }); +}); diff --git a/src/configuration/LogLevelRules/LevelQueryFilter/utils.ts b/src/configuration/LogLevelRules/LevelQueryFilter/utils.ts new file mode 100644 index 00000000..61063198 --- /dev/null +++ b/src/configuration/LogLevelRules/LevelQueryFilter/utils.ts @@ -0,0 +1,126 @@ +interface BuildQueryExprWithLevelFiltersProps { + queryExpr: string; + levelQueryExpr: string; + isUnknownFilter: boolean; + isShiftPressed: boolean; + isQueryContainUnknowFilter?: boolean; +} + +/** + * Constructs a query expression by applying level-based filters to an existing query expression. + * How it works: + * 1. if no query, then use the level query + * 2. if the query exists, then looking for pipe function ----> `filterName1: filterValue1 | filterName2: filterValue2` + * 3. if no pipe, go to step (5) ----> `filterName1: filterValue2` + * 4. if a pipe exists, find the first part of the query ----> `filterName1: filterValue1` + * 5. looking for other level filters in the query ----> `filterName1: filterValue1 level: info` + * 6. If the level filter exists, add a new level filter at the beginning of the query ----> `level: newinfo filterName1: filterValue1 level: info ...` + * 7. if the level filter doesn't exist, add a new pipe, and before pipe add level filter ----> `level: newinfo | filterName1 ...` + * + * @param {BuildQueryExprWithLevelFiltersProps} props - An object containing properties to build the query expression. + * @param {string} props.queryExpr - The initial query expression to which level filters may be applied. + * @param {string} props.levelQueryExpr - The query expression representing the level-based filters. + * @returns {string} The final query expression after applying the level-based filters. Returns the levelQueryExpr if the queryExpr is empty. + */ +export function buildQueryExprWithLevelFilters(props: BuildQueryExprWithLevelFiltersProps) { + if (props.queryExpr.trim().length === 0) { + return props.levelQueryExpr; + } + return buildNonEmptyQuery(props); +} + +function buildNonEmptyQuery(props: BuildQueryExprWithLevelFiltersProps) { + const pipePosition = findPipeSeparatorPosition(props.queryExpr); + if (pipePosition === -1) { + return buildLevelQuery(props); + } else { + const firstPipeQuery = props.queryExpr.slice(0, pipePosition); + const firstPipeModified = buildLevelQuery({ ...props, queryExpr: firstPipeQuery }); + return firstPipeModified + props.queryExpr.slice(pipePosition); + } +} + +function buildLevelQuery({ + queryExpr, + levelQueryExpr, + isUnknownFilter, + isQueryContainUnknowFilter, + isShiftPressed +}: BuildQueryExprWithLevelFiltersProps) { + const isQueryWithFilter = isQueryContainsLevelFilter(queryExpr); + if (!isQueryWithFilter) { + return levelQueryExpr + ' | ' + queryExpr; + } + + if (queryExpr.trim().length === 0) { + return levelQueryExpr; + } + + if (isUnknownFilter) { + return levelQueryExpr; + } + + if (isQueryContainUnknowFilter) { + return levelQueryExpr; + } + + if (isShiftPressed) { + return buildMultiLevelFilter(queryExpr, levelQueryExpr); + } + + return levelQueryExpr; +} + +function buildMultiLevelFilter(queryExpr: string, levelQueryExpr: string) { + if (!queryExpr.includes(levelQueryExpr)) { + return levelQueryExpr + ' OR ' + queryExpr; + } else { + return queryExpr + .split(levelQueryExpr) + .map(trimOR) + .filter(Boolean) + .join(' OR '); + } +} + +function trimOR(query: string) { + let result = query.trim(); + if (result.toLowerCase().startsWith('or')) { + result = result.slice(2).trim(); + } + if (result.toLowerCase().endsWith('or')) { + result = result.slice(0, -2).trim(); + } + return result; +} + +const PIPE_SEPARATOR = '|'; + +function findPipeSeparatorPosition(queryExpr: string): number { + const pipeSeparatorPosition = queryExpr.indexOf(PIPE_SEPARATOR); + if (pipeSeparatorPosition === -1) { + return -1; + } + + let quotesCount = 0; + for (let i = 0; i < queryExpr.length; i++) { + if (queryExpr[i] === '"' && queryExpr[i - 1] !== '\\') { + quotesCount++; + continue; + } + if (queryExpr[i] === PIPE_SEPARATOR && quotesCount % 2 === 0) { + return i; + } + } + + return -1; +} + +// check if queryExpr contains level filter: `level:...` +function isQueryContainsLevelFilter(queryExpr: string): boolean { + const levelFilterRegex = /\s*level\s*:\s*([a-zA-Z0-9_]+)\s*/; + const match = queryExpr.match(levelFilterRegex); + return Boolean(match); +} + + diff --git a/src/configuration/LogLevelRules/const.ts b/src/configuration/LogLevelRules/const.ts index 8c85c1fa..7407b549 100644 --- a/src/configuration/LogLevelRules/const.ts +++ b/src/configuration/LogLevelRules/const.ts @@ -17,6 +17,14 @@ export const LOG_OPERATOR_OPTIONS: LogOperatorOption[] = [ { label: '>', description: 'Greater than', value: LogLevelRuleType.GreaterThan }, ]; +export const OperatorLabels: Record = { + [LogLevelRuleType.Equals]: '=', + [LogLevelRuleType.NotEquals]: '!=', + [LogLevelRuleType.Regex]: '~', + [LogLevelRuleType.LessThan]: '<', + [LogLevelRuleType.GreaterThan]: '>', +}; + export const LOG_LEVEL_OPTIONS: Array> = Array.from( new Set(Object.values(LogLevel)) ).map((level) => ({ @@ -24,6 +32,28 @@ export const LOG_LEVEL_OPTIONS: Array> = Array.from( value: level as LogLevel, })); +export const UNIQ_LOG_LEVEL = { + [LogLevel.critical]: LogLevel.critical, + [LogLevel.error]: LogLevel.error, + [LogLevel.warning]: LogLevel.warning, + [LogLevel.info]: LogLevel.info, + [LogLevel.debug]: LogLevel.debug, + [LogLevel.trace]: LogLevel.trace, + [LogLevel.unknown]: LogLevel.unknown, +} as const; + +export type UniqLogLevelKeys = (typeof UNIQ_LOG_LEVEL)[keyof typeof UNIQ_LOG_LEVEL]; + +export const possibleLogValueByLevelType = Object.keys(LogLevel).reduce((acc, possibleValue) => { + const levelName = LogLevel[possibleValue as LogLevel]; + if (!acc[levelName]) { + acc[levelName] = []; + } + + acc[levelName].push(possibleValue as string); + return acc; +}, {} as Record); + export const LOG_LEVEL_COLOR = { [LogLevel.critical]: colors[7], [LogLevel.error]: colors[4], diff --git a/yarn.lock b/yarn.lock index 97f1bf62..f4ca861a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2149,7 +2149,7 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.20.tgz#1ca77361d7363432d29f5e55950d9ec1e1c6ea93" integrity sha512-H3MHACvFUEiujabxhaI/ImO6gUrd8oOurg7LQtS7mbwIXA/cUqWrvBsaeJ23aZEPk1TAYkurjfMbSELfoCXlGA== -"@types/lodash@^4.17.5": +"@types/lodash@^4.17.21": version "4.17.21" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.21.tgz#b806831543d696b14f8112db600ea9d3a1df6ea4" integrity sha512-FOvQ0YPD5NOfPgMzJihoT+Za5pdkDJWcbpuj1DjaKZIr/gxodQjY/uWEFlTNqW2ugXHUiL8lRQgw63dzKHZdeQ==