Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

## tip

* FEATURE: add opt-in Grafana dataplane log format support. When enabled in datasource settings, log frames use standard field names (`timestamp`, `body`) and `DataFrameType.LogLines` metadata, enabling automatic `${labelKey}` variables in correlations.

## v0.26.3

* BUGFIX: fix time range provided in `field_names` and `field_values` requests. Instead of being rounded to a 24-hour time range, the selected time range is now provided. See [pr #581](https://github.com/VictoriaMetrics/victorialogs-datasource/pull/581).
Expand Down
15 changes: 15 additions & 0 deletions src/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ datasources:
url: http://victorialogs:9428
isDefault: true
jsonData:
# Enable Grafana dataplane log format for correlations support
#useDataplaneFormat: true
# Multitenancy settings, see https://docs.victoriametrics.com/victorialogs/#multitenancy
# to use the multitenancy, uncomment lines below: AccountID and ProjectID
multitenancyHeaders:
Expand Down Expand Up @@ -165,6 +167,19 @@ Where:
* `operator` is the comparison operator to use, such as `equals`, `notEquals`, `regex`, `lessThan`, `greaterThan`.
* `enabled` is a boolean flag to enable or disable the rule. Defaults to `true` if omitted.

### Dataplane format

The **Use dataplane format** toggle in the datasource settings enables the [Grafana dataplane log format](https://grafana.com/developers/dataplane/). When enabled, log frames use standard field names (`timestamp`, `body` instead of `Time`, `Line`) and set `DataFrameType.LogLines` metadata. This makes all label keys automatically available as `${labelKey}` variables in [correlations](https://grafana.com/docs/grafana/latest/administration/correlations/), similar to how Loki works with its dataplane support.

**Note:** Enabling this setting may break existing client-side transformations that reference the `Time` or `Line` field names.

To enable via provisioning:

```yaml
jsonData:
useDataplaneFormat: true
```

### Variables
VictoriaLogs datasource supports [variables](https://grafana.com/docs/grafana/latest/variables/) in queries.

Expand Down
22 changes: 21 additions & 1 deletion src/configuration/LogsSettings.tsx
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import React, { SyntheticEvent, useMemo } from 'react';

import { SelectableValue } from '@grafana/data';
import { InlineField, Input, Stack, Text } from '@grafana/ui';
import { InlineField, InlineSwitch, Input, Stack, Text } from '@grafana/ui';

import { Options } from '../types';

Expand Down Expand Up @@ -53,6 +53,26 @@ export const LogsSettings = (props: PropsConfigEditor) => {
/>
</InlineField>
</div>
<div className='gf-form max-width-30'>
<InlineField
label='Use dataplane format'
labelWidth={28}
tooltip='Use Grafana dataplane log format (timestamp/body field names, DataFrameType.LogLines). Enables automatic label variables in correlations. May break existing client-side transformations that reference Time/Line field names.'
>
<InlineSwitch
value={options.jsonData.useDataplaneFormat}
onChange={(e) => {
onOptionsChange({
...options,
jsonData: {
...options.jsonData,
useDataplaneFormat: e.currentTarget.checked,
},
});
}}
/>
</InlineField>
</div>
</div>
</Stack>
);
Expand Down
5 changes: 4 additions & 1 deletion src/datasource.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ export class VictoriaLogsDatasource
queryBuilderLimits?: QueryBuilderLimits;
logLevelRules: LogLevelRule[];
multitenancyHeaders?: MultitenancyHeaders;
useDataplaneFormat: boolean;

constructor(
instanceSettings: DataSourceInstanceSettings<Options>,
Expand All @@ -120,6 +121,7 @@ export class VictoriaLogsDatasource
this.queryBuilderLimits = settingsData.queryBuilderLimits;
this.logLevelRules = settingsData.logLevelRules || [];
this.multitenancyHeaders = this.parseMultitenancyHeaders(settingsData.multitenancyHeaders);
this.useDataplaneFormat = settingsData.useDataplaneFormat ?? false;
}

query(request: DataQueryRequest<Query>): Observable<DataQueryResponse> {
Expand Down Expand Up @@ -158,7 +160,8 @@ export class VictoriaLogsDatasource
response,
fixedRequest,
this.derivedFields ?? [],
this.getActiveLevelRules()
this.getActiveLevelRules(),
this.useDataplaneFormat
)
)
);
Expand Down
35 changes: 26 additions & 9 deletions src/transformers/frameProcessors/streamFrameProcessor.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { DataFrame, QueryResultMeta } from '@grafana/data';
import { DataFrame, DataFrameType, QueryResultMeta } from '@grafana/data';

import { LogLevelRule } from '../../configuration/LogLevelRules/types';
import { getHighlighterExpressionsFromQuery } from '../../queryUtils';
Expand All @@ -7,19 +7,20 @@ import { getDerivedFields } from '../fields/derivedField';
import { getStreamFields } from '../fields/labelField';
import { addLevelField } from '../fields/levelField';
import { getStreamIds } from '../fields/streamUtils';
import { ANNOTATIONS_REF_ID } from '../types';
import { ANNOTATIONS_REF_ID, FrameField } from '../types';
import { dataFrameHasError, setFrameMeta } from '../utils/frame/frameUtils';

export function processStreamsFrames(
frames: DataFrame[],
queryMap: Map<string, Query>,
derivedFieldConfigs: DerivedFieldConfig[],
logLevelRules: LogLevelRule[]
logLevelRules: LogLevelRule[],
useDataplaneFormat = false,
): DataFrame[] {
return frames.map((frame) => {
const query = frame.refId !== undefined ? queryMap.get(frame.refId) : undefined;
const isAnnotations = query?.refId === ANNOTATIONS_REF_ID;
return processStreamFrame(frame, query, derivedFieldConfigs, logLevelRules, isAnnotations);
return processStreamFrame(frame, query, derivedFieldConfigs, logLevelRules, isAnnotations, useDataplaneFormat);
});
}

Expand All @@ -28,7 +29,8 @@ function processStreamFrame(
query: Query | undefined,
derivedFieldConfigs: DerivedFieldConfig[],
logLevelRules: LogLevelRule[],
transformLabels = false
transformLabels = false,
useDataplaneFormat = false,
): DataFrame {
const custom: Record<string, string> = {
...frame.meta?.custom, // keep the original meta.custom
Expand All @@ -55,11 +57,26 @@ function processStreamFrame(
const derivedFields = getDerivedFields(frameWithLevel, derivedFieldConfigs);
const baseFields = getStreamFields(frameWithLevel.fields, transformLabels);

const fields = [...baseFields, ...derivedFields];

if (useDataplaneFormat) {
return {
...frameWithLevel,
fields: fields.map((f) => {
if (f.name === 'Time') { return { ...f, name: 'timestamp' }; }
if (f.name === FrameField.Line) { return { ...f, name: 'body' }; }
return f;
}),
meta: {
...frameWithLevel.meta,
type: DataFrameType.LogLines,
typeVersion: [0, 0],
},
};
}

return {
...frameWithLevel,
fields: [
...baseFields,
...derivedFields
]
fields,
};
}
91 changes: 90 additions & 1 deletion src/transformers/transformBackendResult.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { DataQueryRequest, DataQueryResponse, dateTime, LogLevel } from '@grafana/data';
import { DataFrameType, DataQueryRequest, DataQueryResponse, dateTime, LogLevel } from '@grafana/data';

import { LogLevelRule, LogLevelRuleType } from '../configuration/LogLevelRules/types';
import { DerivedFieldConfig, Query, QueryType } from '../types';
Expand Down Expand Up @@ -187,6 +187,95 @@ describe('transformBackendResult', () => {
]);
});

it('should use dataplane format when useDataplaneFormat is true', () => {
const response = {
'data': [
{
'refId': 'A',
'meta': {
'typeVersion': [0, 0]
},
'fields': [
{
'name': 'Time',
'type': 'time',
'typeInfo': { 'frame': 'time.Time' },
'config': {},
'values': [1760598702731],
'entities': {},
'nanos': [713000]
},
{
'name': 'Line',
'type': 'string',
'typeInfo': { 'frame': 'string' },
'config': {},
'values': ['starting application'],
'entities': {}
},
{
'name': 'labels',
'type': 'other',
'typeInfo': { 'frame': 'json.RawMessage' },
'config': {},
'values': [labels[0]],
'entities': {}
}
],
'length': 1
}
],
'state': 'Done'
} as DataQueryResponse;
const request = {
'app': 'dashboard',
'requestId': 'SQR100',
'timezone': 'browser',
'range': {
'to': '2025-10-16T07:28:02.475Z',
'from': '2025-10-16T01:28:02.475Z',
'raw': { 'from': 'now-6h', 'to': 'now' }
},
'interval': '20s',
'intervalMs': 20000,
'targets': [
{
'datasource': { 'type': 'victoriametrics-logs-datasource', 'uid': 'bexw8wod6s4jke' },
'editorMode': 'code',
'expr': '*',
'queryType': 'instant',
'refId': 'A',
'maxLines': 1000
}
],
'maxDataPoints': 913,
'scopedVars': {
'__sceneObject': { 'text': '__sceneObject' },
'__interval': { 'text': '20s', 'value': '20s' },
'__interval_ms': { 'text': '20000', 'value': 20000 }
},
'startTime': 1760599682628,
'rangeRaw': { 'from': 'now-6h', 'to': 'now' },
'dashboardUID': '886b7b9f-97a7-47ee-93b6-9ec7342f6d3e',
'panelId': 1,
'panelName': 'New panel',
'panelPluginId': 'table',
'dashboardTitle': 'double label info'
} as unknown as DataQueryRequest<Query>;
const result = transformBackendResult(response, request, [], [], true);
const frame = result.data[0];

// Field names should be renamed to dataplane format
expect(frame.fields[0].name).toBe('timestamp');
expect(frame.fields[1].name).toBe('body');
expect(frame.fields[2].name).toBe('labels');
expect(frame.fields[3].name).toBe('detected_level');

// Meta should have dataplane type
expect(frame.meta?.type).toBe(DataFrameType.LogLines);
expect(frame.meta?.typeVersion).toStrictEqual([0, 0]);
});

it('should parse level according to rules, apply the origin level labels then rule labels', () => {
const extendedLabels = labels.map((l, index) => {
if (index > 2) {
Expand Down
3 changes: 2 additions & 1 deletion src/transformers/transformBackendResult.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export function transformBackendResult(
request: DataQueryRequest<Query>,
derivedFieldConfigs: DerivedFieldConfig[],
logLevelRules: LogLevelRule[],
useDataplaneFormat = false,
): DataQueryResponse {
const { data, errors, ...rest } = response;
const queries = request.targets;
Expand Down Expand Up @@ -44,7 +45,7 @@ export function transformBackendResult(
data: [
...processMetricRangeFrames(metricRangeFrames, request.targets, request.range.from.valueOf(), request.range.to.valueOf()),
...processMetricInstantFrames(metricInstantFrames),
...processStreamsFrames(streamsFrames, queryMap, derivedFieldConfigs, logLevelRules),
...processStreamsFrames(streamsFrames, queryMap, derivedFieldConfigs, logLevelRules, useDataplaneFormat),
...processHistogramFrames(histogramFrames, request.panelPluginId),
],
};
Expand Down
1 change: 1 addition & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ export interface Options extends DataSourceJsonData {
logLevelRules?: LogLevelRule[];
multitenancyHeaders?: Partial<Record<TenantHeaderNames, string>>;
vmuiUrl?: string;
useDataplaneFormat?: boolean;
}

export const QUERY_DIRECTION = {
Expand Down