add support for configuring log level using custom rules (#301)

* style: unify settings titles and format code

* feat: add log level settings

* refactor: rename `field` to `fields` in hits request

---------

Co-authored-by: Dmytro Kozlov <[email protected]>

Author: Loori-R

CHANGELOG.md

@@ -2,7 +2,8 @@
 
 ## tip
 
-* BUGFIX: fix an issue when Grafana decides that the response is not a wide series and shows the error "input data must be a wide series but got type not (input refid)" . See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/302).
+* FEATURE: add support for configuring log level using custom rules. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/294).
+* BUGFIX: fix an issue when Grafana decides that the response is not a wide series and shows the error "input data must be a wide series but got type not (input refid)". See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/302).
 
 ## v0.16.3
 

pkg/plugin/query.go

@@ -53,7 +53,7 @@ type Query struct {
 	IntervalMs   int64     `json:"intervalMs"`
 	MaxLines     int       `json:"maxLines"`
 	Step         string    `json:"step"`
-	Field        string    `json:"field"`
+	Fields       []string  `json:"fields"`
 	QueryType    QueryType `json:"queryType"`
 	url          *url.URL
 	ForAlerting  bool `json:"-"`
@@ -256,7 +256,9 @@ func (q *Query) histQueryURL(queryParams url.Values, minInterval time.Duration)
 	values.Set("start", strconv.FormatInt(q.TimeRange.From.Unix(), 10))
 	values.Set("end", strconv.FormatInt(q.TimeRange.To.Unix(), 10))
 	values.Set("step", step)
-	values.Set("field", q.Field)
+	for _, f := range q.Fields {
+		values.Add("field", f)
+	}
 
 	q.url.RawQuery = values.Encode()
 	return q.url.String()

pkg/plugin/query_test.go

@@ -296,7 +296,7 @@ func TestQuery_getQueryURL(t *testing.T) {
 				rawURL:      "http://127.0.0.1:9429",
 				queryParams: "",
 			},
-			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=&start=1609459200&step=15s",
+			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=&start=1609459200&step=15s",
 			wantErr: false,
 		},
 		{
@@ -315,7 +315,7 @@ func TestQuery_getQueryURL(t *testing.T) {
 				rawURL:      "http://127.0.0.1:9429",
 				queryParams: "",
 			},
-			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=&start=1609459200&step=15s",
+			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=&start=1609459200&step=15s",
 			wantErr: false,
 		},
 		{
@@ -334,7 +334,7 @@ func TestQuery_getQueryURL(t *testing.T) {
 				rawURL:      "http://127.0.0.1:9429",
 				queryParams: "",
 			},
-			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=&start=1609459200&step=15s",
+			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=&start=1609459200&step=15s",
 			wantErr: false,
 		},
 		{
@@ -353,7 +353,7 @@ func TestQuery_getQueryURL(t *testing.T) {
 				rawURL:      "http://127.0.0.1:9429",
 				queryParams: "",
 			},
-			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s&start=1609459200&step=15s",
+			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s&start=1609459200&step=15s",
 			wantErr: false,
 		},
 		{
@@ -372,7 +372,7 @@ func TestQuery_getQueryURL(t *testing.T) {
 				rawURL:      "http://127.0.0.1:9429",
 				queryParams: "",
 			},
-			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
+			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
 			wantErr: false,
 		},
 		{
@@ -391,7 +391,7 @@ func TestQuery_getQueryURL(t *testing.T) {
 				rawURL:      "http://127.0.0.1:9429",
 				queryParams: "",
 			},
-			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
+			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
 			wantErr: false,
 		},
 		{
@@ -410,7 +410,7 @@ func TestQuery_getQueryURL(t *testing.T) {
 				rawURL:      "http://127.0.0.1:9429",
 				queryParams: "",
 			},
-			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+and+syslog&start=1609459200&step=15s",
+			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+and+syslog&start=1609459200&step=15s",
 			wantErr: false,
 		},
 		{
@@ -429,7 +429,7 @@ func TestQuery_getQueryURL(t *testing.T) {
 				rawURL:      "http://127.0.0.1:9429",
 				queryParams: "",
 			},
-			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
+			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
 			wantErr: false,
 		},
 		{
@@ -448,7 +448,7 @@ func TestQuery_getQueryURL(t *testing.T) {
 				rawURL:      "http://127.0.0.1:9429",
 				queryParams: "",
 			},
-			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
+			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
 			wantErr: false,
 		},
 		{
@@ -467,7 +467,7 @@ func TestQuery_getQueryURL(t *testing.T) {
 				rawURL:      "http://127.0.0.1:9429",
 				queryParams: "",
 			},
-			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=%2A+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
+			want:    "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&query=%2A+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
 			wantErr: false,
 		},
 	}

src/README.md

@@ -81,6 +81,31 @@ And apply `Transformations` by labels:
 
 <img alt="Transformations" src="https://github.com/VictoriaMetrics/victorialogs-datasource/blob/main/src/img/panel_table_transformation.png?raw=true">
 
+### Log level rules
+
+The **Log level rules** section in the datasource configuration allows you to assign log levels based on custom field conditions. This helps classify logs dynamically (e.g., as `error`, `info`, `debug`, etc.) using rules you define.
+
+#### How to use
+
+1. Open the datasource settings.
+
+2. Scroll to the **Log level rules** section.
+
+3. Click **"Add rule"** to define a new rule.
+
+4. For each rule, configure the following:
+
+    * **Enable switch** – enable or disable the rule.
+    * **Field name** – the log field the condition will evaluate.
+    * **Operator** – choose from: `Equals`, `Not equal`, `Matches regex`, `Less than`, `Greater than`
+    * **Value** – the value to compare the field against.
+    * **Log level** – level to assign if the condition matches: `critical`, `warning`, `error`, `info`, `debug`, `trace`, `unknown`
+    * **Delete button** – remove the rule.
+
+5.  After adding or editing rules, click **"Save & test"** to apply the changes.
+
+**Rule priority**: If multiple rules match a log entry, the **first matching rule** (top to bottom) takes precedence.
+
 ## License
 
 This project is licensed under

src/backendResultTransformer.ts

@@ -3,17 +3,28 @@ import {
   DataQueryError,
   DataQueryRequest,
   DataQueryResponse,
+  Field,
   FieldType,
-  isDataFrame,
+  LogLevel,
   QueryResultMeta,
+  isDataFrame,
 } from '@grafana/data';
 
+import { LogLevelRule } from "./configuration/LogLevelRules/types";
+import { resolveLogLevel } from "./configuration/LogLevelRules/utils";
 import { getDerivedFields } from './getDerivedFields';
 import { makeTableFrames } from './makeTableFrames';
 import { getHighlighterExpressionsFromQuery } from './queryUtils';
 import { dataFrameHasError } from './responseUtils';
 import { DerivedFieldConfig, Query, QueryType } from './types';
 
+const ANNOTATIONS_REF_ID = 'Anno';
+
+enum FrameField {
+  Labels = 'labels',
+  Level = 'level'
+}
+
 function isMetricFrame(frame: DataFrame): boolean {
   return frame.fields.every((field) => field.type === FieldType.time || field.type === FieldType.number);
 }
@@ -29,22 +40,67 @@ function setFrameMeta(frame: DataFrame, meta: QueryResultMeta): DataFrame {
   };
 }
 
+function addLevelField(frame: DataFrame, rules: LogLevelRule[]): DataFrame {
+  const rows = frame.length ?? frame.fields[0]?.values.length ?? 0;
+  const labelsField = frame.fields.find(f => f.name === FrameField.Labels);
+
+  const levelValues: LogLevel[] = Array.from({ length: rows }, (_, idx) => {
+    const labels = (labelsField?.values[idx] ?? {}) as Record<string, any>;
+    return resolveLogLevel(labels, rules);
+  });
+
+  const levelField: Field<LogLevel> = {
+    name: FrameField.Level,
+    type: FieldType.string,
+    config: {},
+    values: levelValues,
+  };
+
+  return { ...frame, fields: [...frame.fields, levelField] };
+}
+
+function transformDashboardLabelField(field: Field): Field {
+  if (field.name !== FrameField.Labels) {
+    return field;
+  }
+
+  return {
+    ...field,
+    values: field.values.map((value) => {
+      return Object.entries(value).map(([key, val]) => {
+        return `${key}: ${JSON.stringify(val)}`;
+      });
+    }),
+  };
+}
+
+function getStreamFields(fields: Field[], transformLabels: boolean): Field[] {
+  if (!transformLabels) {
+    return fields;
+  }
+
+  return fields.map(transformDashboardLabelField);
+}
+
 function processStreamsFrames(
   frames: DataFrame[],
   queryMap: Map<string, Query>,
   derivedFieldConfigs: DerivedFieldConfig[],
+  logLevelRules: LogLevelRule[]
 ): DataFrame[] {
   return frames.map((frame) => {
     const query = frame.refId !== undefined ? queryMap.get(frame.refId) : undefined;
-    if (query?.refId === "Anno") {return processDashboardStreamFrame(frame, query, derivedFieldConfigs);}
-    return processStreamFrame(frame, query, derivedFieldConfigs);
+    const isAnnotations = query?.refId === ANNOTATIONS_REF_ID
+    return processStreamFrame(frame, query, derivedFieldConfigs, logLevelRules, isAnnotations);
   });
 }
 
 function processStreamFrame(
   frame: DataFrame,
   query: Query | undefined,
-  derivedFieldConfigs: DerivedFieldConfig[]
+  derivedFieldConfigs: DerivedFieldConfig[],
+  logLevelRules: LogLevelRule[],
+  transformLabels = false
 ): DataFrame {
   const custom: Record<string, string> = {
     ...frame.meta?.custom, // keep the original meta.custom
@@ -61,55 +117,18 @@ function processStreamFrame(
     custom,
   };
 
-  const newFrame = setFrameMeta(frame, meta);
-  const derivedFields = getDerivedFields(newFrame, derivedFieldConfigs);
-  return {
-    ...newFrame,
-    fields: [...newFrame.fields, ...derivedFields],
-  };
-}
+  const frameWithMeta = setFrameMeta(frame, meta);
+  const frameWithLevel = addLevelField(frameWithMeta, logLevelRules);
 
-function processDashboardStreamFrame(
-  frame: DataFrame,
-  query: Query | undefined,
-  derivedFieldConfigs: DerivedFieldConfig[]
-): DataFrame {
-  const custom: Record<string, string> = {
-    ...frame.meta?.custom, // keep the original meta.custom
-  };
-
-  if (dataFrameHasError(frame)) {
-    custom.error = 'Error when parsing some of the logs';
-  }
-
-  const meta: QueryResultMeta = {
-    preferredVisualisationType: 'logs',
-    limit: query?.maxLines,
-    searchWords: query !== undefined ? getHighlighterExpressionsFromQuery(query.expr) : undefined,
-    custom,
-  };
-
-  const newFrame = setFrameMeta(frame, meta);
-  const derivedFields = getDerivedFields(newFrame, derivedFieldConfigs);
+  const derivedFields = getDerivedFields(frameWithLevel, derivedFieldConfigs);
+  const baseFields = getStreamFields(frameWithLevel.fields, transformLabels)
 
   return {
-    ...newFrame,
+    ...frameWithLevel,
     fields: [
-      ...newFrame.fields.map((field) => {
-        if (field.name === 'labels') {
-          return {
-            ...field,
-            values: field.values.map((value) => {
-              return Object.entries(value).map(([key, value]) => {
-                return `${key}: ${JSON.stringify(value)}`;
-              });
-            }),
-          };
-        }
-        return field;
-      }),
-      ...derivedFields,
-    ],
+      ...baseFields,
+      ...derivedFields
+    ]
   };
 }
 
@@ -180,7 +199,8 @@ function improveError(error: DataQueryError | undefined, queryMap: Map<string, Q
 export function transformBackendResult(
   response: DataQueryResponse,
   request: DataQueryRequest,
-  derivedFieldConfigs: DerivedFieldConfig[]
+  derivedFieldConfigs: DerivedFieldConfig[],
+  logLevelRules: LogLevelRule[],
 ): DataQueryResponse {
   const { data, errors, ...rest } = response;
   const queries = request.targets;
@@ -208,7 +228,7 @@ export function transformBackendResult(
     data: [
       ...processMetricRangeFrames(metricRangeFrames),
       ...processMetricInstantFrames(metricInstantFrames),
-      ...processStreamsFrames(streamsFrames, queryMap, derivedFieldConfigs),
+      ...processStreamsFrames(streamsFrames, queryMap, derivedFieldConfigs, logLevelRules),
     ],
   };
 }

src/configuration/ConfigEditor.tsx

@@ -1,16 +1,17 @@
 import React from 'react';
 import { gte } from 'semver';
 
-import { DataSourcePluginOptionsEditorProps, DataSourceSettings } from '@grafana/data';
+import { DataSourcePluginOptionsEditorProps, DataSourceSettings, FeatureToggles } from '@grafana/data';
 import { config } from '@grafana/runtime';
-import { InlineSwitch, InlineField, DataSourceHttpSettings, SecureSocksProxySettings } from "@grafana/ui";
+import { InlineSwitch, InlineField, DataSourceHttpSettings, Space } from "@grafana/ui";
 
 import { Options } from '../types';
 
 import { AlertingSettings } from './AlertingSettings';
 import { DerivedFields } from "./DerivedFields";
 import { HelpfulLinks } from "./HelpfulLinks";
 import { LimitsSettings } from "./LimitSettings";
+import { LogLevelRulesEditor } from "./LogLevelRules/LogLevelRulesEditor";
 import { LogsSettings } from './LogsSettings';
 import { QuerySettings } from './QuerySettings';
 
@@ -41,18 +42,24 @@ const ConfigEditor = (props: Props) => {
         sigV4AuthToggleEnabled={config.sigV4AuthEnabled}
       />
       <AlertingSettings options={options} onOptionsChange={onOptionsChange}/>
-      <QuerySettings
-        maxLines={options.jsonData.maxLines || ''}
-        onMaxLinedChange={(value) => onOptionsChange(setMaxLines(options, value))}
-      />
+
+      <Space v={5} />
+
+      <LimitsSettings {...props}>
+        <QuerySettings
+          maxLines={options.jsonData.maxLines || ''}
+          onMaxLinedChange={(value) => onOptionsChange(setMaxLines(options, value))}
+        />
+      </LimitsSettings>
+
+      <LogsSettings {...props}/>
+
       <DerivedFields
         fields={options.jsonData.derivedFields}
         onChange={(value) => onOptionsChange(setDerivedFields(options, value))}
       />
 
-      <LogsSettings {...props}/>
-
-      <LimitsSettings {...props}/>
+      <LogLevelRulesEditor {...props}/>
 
       {config.featureToggles['secureSocksDSProxyEnabled' as keyof FeatureToggles] && gte(config.buildInfo.version, '10.0.0') && (
         <>