add quick level filter, which allows filtering logs by level according to Log Level Rules and the base level field

Author: arturminchukov

CHANGELOG.md

@@ -5,6 +5,7 @@
 * FEATURE: fetch tenants from the VictoriaLogs backend and allow selecting a tenant in the datasource settings. See [#475](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/475).
 * FEATURE: enable client side caching and make reliable behavior in QueryBuilder filters. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/357).
 * FEATURE: add compatibility with Grafana 10.x and 11.x by using dynamic component loading for Combobox.
+* FEATURE: add quick level filter, which allows filtering logs by level according to `Log Level Rules` and the base level field. It is the first part of the [issue #108](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/108). See [pr #495](https://github.com/VictoriaMetrics/victorialogs-datasource/pull/495).
 
 ## v0.22.4
 

package.json

@@ -34,9 +34,10 @@
     "@testing-library/jest-dom": "6.4.6",
     "@testing-library/react": "16.0.0",
     "@types/jest": "^29.5.14",
-    "@types/lodash": "^4.17.5",
+    "@types/lodash": "^4.17.21",
     "@types/node": "^20.14.9",
     "@types/react": "^19.2.2",
+    "@types/semver": "^7.7.0",
     "@types/testing-library__jest-dom": "6.0.0",
     "@typescript-eslint/eslint-plugin": "^8.26.1",
     "@typescript-eslint/parser": "^8.26.1",
@@ -88,7 +89,7 @@
     "@grafana/ui": "12.2.0",
     "@lezer/common": "^1.2.1",
     "@lezer/lr": "^1.4.1",
-    "@types/semver": "^7.7.0",
+    "lodash": "^4.17.21",
     "react": "18.3.1",
     "react-dom": "18.3.1",
     "rxjs": "^7.8.2",

src/components/QueryEditor/EditorHeader.tsx

@@ -19,7 +19,7 @@ const getStyles = (theme: GrafanaTheme2) => ({
     display: 'flex',
     flexWrap: 'wrap',
     alignItems: 'center',
-    justifyContent: 'flex-end',
+    justifyContent: 'space-between',
     gap: theme.spacing(1),
     minHeight: theme.spacing(4),
   }),

src/components/QueryEditor/QueryEditor.tsx

@@ -3,10 +3,11 @@ import { isEqual } from 'lodash';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 
 import { CoreApp, GrafanaTheme2, LoadingState } from '@grafana/data';
-import { Button, ConfirmModal, useStyles2 } from '@grafana/ui';
+import { Button, ConfirmModal, Stack, useStyles2 } from '@grafana/ui';
 
 import { getQueryExprVariableRegExp } from "../../LogsQL/regExpOperator";
 import { isExprHasStatsPipeFunctions } from "../../LogsQL/statsPipeFunctions";
+import { LevelQueryFilter } from "../../configuration/LogLevelRules/LevelQueryFilter/LeveQueryFilter";
 import { storeKeys } from "../../store/constants";
 import store from "../../store/store";
 import { Query, QueryEditorMode, QueryType, VictoriaLogsQueryEditorProps } from "../../types";
@@ -90,25 +91,29 @@ const QueryEditor = React.memo<VictoriaLogsQueryEditorProps>((props) => {
       />
       <div className={styles.wrapper}>
         <EditorHeader>
-          {showStatsWarn && (<QueryEditorStatsWarn queryType={query.queryType}/>)}
-          <QueryEditorHelp />
-          <VmuiLink
-            query={query}
-            panelData={data}
-            datasource={datasource}
-          />
-          <QueryEditorModeToggle mode={editorMode} onChange={onEditorModeChange}/>
-          {app !== CoreApp.Explore && app !== CoreApp.Correlations && (
-            <Button
-              variant={dataIsStale ? 'primary' : 'secondary'}
-              size="sm"
-              onClick={onRunQuery}
-              icon={data?.state === LoadingState.Loading ? 'fa fa-spinner' : undefined}
-              disabled={data?.state === LoadingState.Loading}
-            >
-              {queries && queries.length > 1 ? `Run queries` : `Run query`}
-            </Button>
-          )}
+          {app === CoreApp.Explore &&
+            <LevelQueryFilter logLevelRules={datasource.logLevelRules} query={query} onChange={onChange}/>}
+          <Stack direction={"row"} justifyContent={"flex-end"} alignItems={"center"} >
+            {showStatsWarn && (<QueryEditorStatsWarn queryType={query.queryType}/>)}
+            <QueryEditorHelp/>
+            <VmuiLink
+              query={query}
+              panelData={data}
+              datasource={datasource}
+            />
+            <QueryEditorModeToggle mode={editorMode} onChange={onEditorModeChange}/>
+            {app !== CoreApp.Explore && app !== CoreApp.Correlations && (
+              <Button
+                variant={dataIsStale ? 'primary' : 'secondary'}
+                size="sm"
+                onClick={onRunQuery}
+                icon={data?.state === LoadingState.Loading ? 'fa fa-spinner' : undefined}
+                disabled={data?.state === LoadingState.Loading}
+              >
+                {queries && queries.length > 1 ? `Run queries` : `Run query`}
+              </Button>
+            )}
+          </Stack>
         </EditorHeader>
         <div className="flex-grow-1">
           {editorMode === QueryEditorMode.Builder ? (

src/components/QueryEditor/QueryHints/hints/types.ts

@@ -0,0 +1,14 @@
+export interface QueryHints {
+  sections: QueryHintSection[];
+}
+
+export interface QueryHintSection {
+  title: string;
+  hints: QueryHint[];
+}
+
+export interface QueryHint {
+  title: string;
+  queryExpr: string;
+  description?: string;
+}

src/components/QueryEditor/QueryHints/hints/useLevelQueryHintSection.ts

@@ -0,0 +1,56 @@
+import { groupBy } from 'lodash';
+import { useMemo } from "react";
+
+import { LogLevel } from "@grafana/data";
+
+import {
+  OperatorLabels,
+  possibleLogValueByLevelType,
+  UNIQ_LOG_LEVEL,
+  UniqLogLevelKeys
+} from "../../../../configuration/LogLevelRules/const";
+import { LogLevelRule } from "../../../../configuration/LogLevelRules/types";
+
+import { QueryHint, QueryHintSection } from "./types";
+
+export const useLevelQueryHintSection = (levelRules: LogLevelRule[]): QueryHintSection => {
+  return useMemo(() => {
+    const enabledLevelRules = levelRules.filter(rule => rule.enabled);
+    const groupedByLevelRules = groupBy(enabledLevelRules, 'level');
+    const levelFilters = Object
+      .values(UNIQ_LOG_LEVEL)
+      .filter(val => val !== LogLevel.unknown)
+      .reduce((acc, logLevel) => {
+        acc[logLevel] = groupedByLevelRules[logLevel] || [];
+        return acc;
+      }, {} as Record<UniqLogLevelKeys, LogLevelRule[]>);
+
+    const hints = Object
+      .entries(levelFilters)
+      .map(([ruleLevel, rule]): QueryHint => {
+        const levelKey = ruleLevel as UniqLogLevelKeys;
+        const queryExprByRules = rule.map(r => `${r.field}:${OperatorLabels[r.operator]}"${r.value}"`);
+        const possibleLevelValues = possibleLogValueByLevelType[levelKey].map(value => `"${value}"`).join(',');
+        const queryExprByLevel = `level:contains_common_case(${possibleLevelValues})`;
+        const queryParts = [queryExprByLevel];
+        if (queryExprByRules.length > 0) {
+          queryParts.push(...queryExprByRules);
+        }
+        const expr = queryParts.join(' OR ');
+        return {
+          title: levelKey,
+          queryExpr: expr
+        };
+      });
+
+    hints.push({
+      title: LogLevel.unknown,
+      queryExpr: `!(${hints.map(hint => hint.queryExpr).join(' OR ')})`
+    });
+
+    return {
+      title: 'Filter by log level',
+      hints
+    };
+  }, [levelRules]);
+};

src/configuration/LogLevelRules/LevelQueryFilter/LeveQueryFilter.tsx

@@ -0,0 +1,58 @@
+import React, { MouseEvent, useCallback, useMemo } from "react";
+
+
+import { LogLevel } from "@grafana/data";
+import { Stack } from "@grafana/ui";
+
+import { useLevelQueryHintSection } from "../../../components/QueryEditor/QueryHints/hints/useLevelQueryHintSection";
+import { Query } from "../../../types";
+import { LogLevelRule } from "../types";
+
+import { LevelFilterButton } from "./LevelFilterButton";
+import { buildQueryExprWithLevelFilters } from "./utils";
+
+
+interface Props {
+  logLevelRules: LogLevelRule[];
+  query: Query;
+  onChange: (value: Query) => void;
+}
+
+export const LevelQueryFilter = ({ logLevelRules, query, onChange }: Props) => {
+  const levelQueryHintSection = useLevelQueryHintSection(logLevelRules);
+
+  const unknownLevelFilter = useMemo(() => levelQueryHintSection.hints.find(hint => hint.title === LogLevel.unknown), [levelQueryHintSection]);
+  const isQueryContainUnknowFilter = unknownLevelFilter && query.expr.includes(unknownLevelFilter.queryExpr);
+
+  const handleClick = useCallback((e: MouseEvent<HTMLButtonElement>, levelQueryExpr: string, title: string) => {
+    const isShiftPressed = e.shiftKey;
+    const isUnknownFilter = title === LogLevel.unknown;
+    const queryExpr = buildQueryExprWithLevelFilters({
+      queryExpr: query.expr,
+      levelQueryExpr,
+      isShiftPressed,
+      isQueryContainUnknowFilter,
+      isUnknownFilter
+    })
+    onChange({ ...query, expr: queryExpr });
+  }, [isQueryContainUnknowFilter, onChange, query]);
+
+  return (
+    <Stack direction={"row"} justifyContent={"flex-start"} alignItems={"center"} wrap={"wrap"}>
+      {levelQueryHintSection.hints.map(({ title, queryExpr }) => {
+        const isNegativeStart = query.expr.startsWith('!(');
+        const isSelected = (isNegativeStart && title === LogLevel.unknown)
+          || (!isNegativeStart && query.expr.includes(queryExpr));
+        return (
+          <LevelFilterButton
+            key={title}
+            onClick={(e: MouseEvent<HTMLButtonElement>) => handleClick(e, queryExpr, title)}
+            level={title as LogLevel}
+            label={title}
+            isSelected={isSelected}
+          />
+        )
+      })}
+    </Stack>
+  );
+};

src/configuration/LogLevelRules/LevelQueryFilter/LevelFilterButton.tsx

@@ -0,0 +1,48 @@
+import { css } from "@emotion/css";
+import React, { MouseEvent } from "react";
+
+import { LogLevel } from "@grafana/data";
+import { Button, Stack, useStyles2 } from "@grafana/ui";
+
+import { LOG_LEVEL_COLOR } from "../const";
+
+export interface LevelFilterButtonProps {
+  onClick: (e: MouseEvent<HTMLButtonElement>) => void;
+  label: string;
+  level: LogLevel;
+  isSelected?: boolean;
+}
+
+export const LevelFilterButton = ({ onClick, label, level, isSelected }: LevelFilterButtonProps) => {
+  const styles = useStyles2(getStyles);
+  return (
+    <Stack direction={"row"}>
+      <Button
+        onClick={onClick}
+        variant={"secondary"}
+        size={"sm"}
+        type={"button"}
+        style={{ opacity: isSelected ? 1 : 0.5, userSelect: 'none' }}
+        title={"Use 'shift' to select several levels"}
+      >
+        <div className={styles.colorCircle} style={{ backgroundColor: getLogLevelColor(level) }}/>
+        {label}
+      </Button>
+    </Stack>
+  );
+};
+
+const getLogLevelColor = (level: LogLevel): string => {
+  return LOG_LEVEL_COLOR[level] || LOG_LEVEL_COLOR[LogLevel.unknown];
+};
+
+const getStyles = () => {
+  return {
+    colorCircle: css({
+      width: 12,
+      height: 12,
+      borderRadius: '50%',
+      marginRight: 5,
+    }),
+  };
+};

src/configuration/LogLevelRules/LevelQueryFilter/utils.ts

@@ -0,0 +1,110 @@
+interface BuildQueryExprWithLevelFiltersProps {
+  queryExpr: string;
+  levelQueryExpr: string;
+  isUnknownFilter: boolean;
+  isShiftPressed: boolean;
+  isQueryContainUnknowFilter?: boolean;
+}
+
+export function buildQueryExprWithLevelFilters(props: BuildQueryExprWithLevelFiltersProps) {
+  if (props.queryExpr.trim().length === 0) {
+    return props.levelQueryExpr;
+  }
+  return buildNonEmptyQuery(props)
+}
+
+function buildNonEmptyQuery(props: BuildQueryExprWithLevelFiltersProps) {
+  const pipePosition = findPipeSeparatorPosition(props.queryExpr);
+  if (pipePosition === -1) {
+    return buildLevelQuery(props);
+  } else {
+    const firstPipeQuery = props.queryExpr.slice(0, pipePosition);
+    const firstPipeModified = buildLevelQuery({ ...props, queryExpr: firstPipeQuery });
+    return firstPipeModified + props.queryExpr.slice(pipePosition);
+  }
+}
+
+function buildLevelQuery({
+  queryExpr,
+  levelQueryExpr,
+  isUnknownFilter,
+  isQueryContainUnknowFilter,
+  isShiftPressed
+}: BuildQueryExprWithLevelFiltersProps) {
+  const isQueryWithFilter = isQueryContainsLevelFilter(queryExpr);
+  if (!isQueryWithFilter) {
+    return levelQueryExpr + ' | ' + queryExpr;
+  }
+
+  if (queryExpr.trim().length === 0) {
+    return levelQueryExpr;
+  }
+
+  if (isUnknownFilter) {
+    return levelQueryExpr;
+  }
+
+  if (isQueryContainUnknowFilter) {
+    return levelQueryExpr;
+  }
+
+  if (isShiftPressed) {
+    return buildMultiLevelFilter(queryExpr, levelQueryExpr);
+  }
+
+  return levelQueryExpr;
+}
+
+function buildMultiLevelFilter(queryExpr: string, levelQueryExpr: string) {
+  if (!queryExpr.includes(levelQueryExpr)) {
+    return levelQueryExpr + ' OR ' + queryExpr;
+  } else {
+    return queryExpr
+      .split(levelQueryExpr)
+      .map(trimOR)
+      .filter(Boolean)
+      .join(' OR ');
+  }
+}
+
+function trimOR(query: string) {
+  let result = query.trim();
+  if (result.toLowerCase().startsWith('or')) {
+    result = result.slice(2).trim();
+  }
+  if (result.toLowerCase().endsWith('or')) {
+    result = result.slice(0, -2).trim();
+  }
+  return result;
+}
+
+const PIPE_SEPARATOR = '|';
+
+function findPipeSeparatorPosition(queryExpr: string): number {
+  const pipeSeparatorPosition = queryExpr.indexOf(PIPE_SEPARATOR);
+  if (pipeSeparatorPosition === -1) {
+    return -1;
+  }
+
+  let quotesCount = 0;
+  for (let i = 0; i < queryExpr.length; i++) {
+    if (queryExpr[i] === '"' && queryExpr[i - 1] !== '\\') {
+      quotesCount++;
+      continue;
+    }
+    if (queryExpr[i] === PIPE_SEPARATOR && quotesCount % 2 === 0) {
+      return i;
+    }
+  }
+
+  return -1;
+}
+
+// check if queryExpr contains level filter: `level:...`
+function isQueryContainsLevelFilter(queryExpr: string): boolean {
+  const levelFilterRegex = /\s*level\s*:\s*([a-zA-Z0-9_]+)\s*/;
+  const match = queryExpr.match(levelFilterRegex);
+  return Boolean(match);
+}
+
+