Skip to content

Commit c06ecdb

Browse files
arturminchukovarturminchukov
authored
460 live tailing logs in grafanas explore page does not work (#462)
* fix an issue where live tailing logs in Grafana's explore page does not work * normalize `all value` in interpolation of a query: - if allValue is default '.*', convert to '*' for non-regex usage - if regex allValue is default '*', convert to '.*' for regex usage * add a word boundary for the replaceAllOptionInQuery function and escape values in the regexp with double quotes
1 parent f79029d commit c06ecdb

File tree

6 files changed

+210
-118
lines changed

6 files changed

+210
-118
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
* BUGFIX: fix an issue where an empty textbox variable was incorrectly interpolated. See [#454](https://github.com/VictoriaMetrics/victorialogs-datasource/pull/454).
66
* BUGFIX: fix a `glob` package vulnerability [CVE-2025-64756](https://github.com/advisories/GHSA-5j98-mcp5-4vw2).
7+
* BUGFIX: fix an issue where live tailing logs in Grafana's explore page does not work. See [#460](https://github.com/VictoriaMetrics/victorialogs-datasource/pull/460).
78

89
## v0.22.1
910

‎src/datasource.test.ts‎

Lines changed: 1 addition & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { AdHocVariableFilter, QueryVariableModel } from '@grafana/data';
1+
import { AdHocVariableFilter } from '@grafana/data';
22
import { TemplateSrv } from "@grafana/runtime";
33

44
import { createDatasource } from "./__mocks__/datasource";
@@ -298,84 +298,4 @@ describe('VictoriaLogsDatasource', () => {
298298
expect(result).toStrictEqual('foo:in($var1) bar:in(*) | *');
299299
})
300300
});
301-
302-
describe('replaceAllOption', () => {
303-
it('should replace variables with custom allValue', () => {
304-
const queryExpr = 'namespace: $namespace';
305-
const variable: QueryVariableModel = {
306-
name: 'namespace',
307-
allValue: 'all_namespaces',
308-
options: [],
309-
} as any;
310-
311-
const result = ds.replaceAllOption(queryExpr, variable);
312-
expect(result).toBe('namespace: all_namespaces');
313-
});
314-
315-
it('should use options list if allValue is not provided and allowCustomValue', () => {
316-
const queryExpr = 'namespace:~"$namespace"';
317-
const variable = {
318-
name: 'namespace',
319-
allValue: null,
320-
allowCustomValue: true,
321-
options: [{ value: 'ns1' }, { value: 'ns2' }],
322-
} as any;
323-
324-
const result = ds.replaceAllOption(queryExpr, variable);
325-
expect(result).toBe('namespace:~"(ns1|ns2)"');
326-
});
327-
328-
it('should use options list if allValue is not provided and query defined', () => {
329-
const queryExpr = 'namespace:~"$namespace"';
330-
const variable = {
331-
name: 'namespace',
332-
allValue: null,
333-
options: [{ value: 'ns1' }, { value: 'ns2' }],
334-
query: {
335-
query: 'filter'
336-
}
337-
} as any;
338-
339-
const result = ds.replaceAllOption(queryExpr, variable);
340-
expect(result).toBe('namespace:~"(ns1|ns2)"');
341-
});
342-
343-
it('should use default wildcard if allValue and options are missing', () => {
344-
const queryExpr = 'namespace:~"$namespace"';
345-
const variable = {
346-
name: 'namespace',
347-
allValue: null,
348-
options: [],
349-
} as any;
350-
351-
const result = ds.replaceAllOption(queryExpr, variable);
352-
expect(result).toBe('namespace:~".*"');
353-
});
354-
355-
it('should replace variables when regex and allowCustomValue are enabled', () => {
356-
const queryExpr = 'namespace:~"$namespace"';
357-
const variable = {
358-
name: 'namespace',
359-
allValue: null,
360-
allowCustomValue: true,
361-
regex: true,
362-
options: [{ value: 'ns1' }, { value: 'ns2' }],
363-
} as any;
364-
365-
const result = ds.replaceAllOption(queryExpr, variable);
366-
expect(result).toBe('namespace:~"(ns1|ns2)"');
367-
});
368-
369-
it('should replace multiple occurrences of the same variable', () => {
370-
const queryExpr = 'namespace:$namespace AND pod:$namespace';
371-
const variable = {
372-
name: 'namespace',
373-
allValue: 'all_namespaces',
374-
options: [],
375-
} as any;
376-
377-
const result = ds.replaceAllOption(queryExpr, variable);
378-
expect(result).toBe('namespace:all_namespaces AND pod:all_namespaces');
379-
});
380-
});
381301
});

‎src/datasource.ts‎

Lines changed: 3 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ import {
5959
ToggleFilterAction,
6060
VariableQuery,
6161
} from './types';
62+
import { replaceAllOptionInQuery } from "./utils/replaceAllOptionInQuery";
6263
import { getMillisecondsFromDuration } from "./utils/timeUtils";
6364
import { VariableSupport } from "./variableSupport/VariableSupport";
6465

@@ -118,7 +119,7 @@ export class VictoriaLogsDatasource
118119
return {
119120
...q,
120121
// to backend sort for limited data to show first logs in the selected time range if the user clicks on the sort button
121-
expr: addSortPipeToQuery(q, sortOrder),
122+
expr: addSortPipeToQuery(q, sortOrder, request.liveStreaming),
122123
maxLines: q.maxLines ?? this.maxLines,
123124
}
124125
});
@@ -288,38 +289,6 @@ export class VictoriaLogsDatasource
288289
return Array.isArray(value) ? value.includes(VARIABLE_ALL_VALUE) : false;
289290
}
290291

291-
/**
292-
* Replaces all occurrences of a variable in the query expression with the following priority:
293-
* 1. Custom All Value
294-
* 2. list of variables (if query defined)
295-
* 3. default '*' and '.*' for regex
296-
*
297-
* @param {string} queryExpr - The query expression where the variable needs to be replaced.
298-
* @param {QueryVariableModel} variable - The variable model containing the variable name, options, and allValue.
299-
* @return {string} - The modified query expression with the variable replaced by its corresponding values or patterns.
300-
*/
301-
replaceAllOption(queryExpr: string, variable: QueryVariableModel): string {
302-
const variableName = variable.name;
303-
304-
// Check if the variable is modified to filter values with query | regexp | allowCustomValue.
305-
// In this case, we need to use a list of options for allValue and regexpAllValue
306-
const isModifiedAllOption = variable.query?.query || variable.allowCustomValue || variable.regex;
307-
let allValue = variable.allValue;
308-
let regexpAllValue = variable.allValue;
309-
if (!allValue && isModifiedAllOption) {
310-
const values = variable.options.map(option => option.value).flat(1);
311-
allValue = values.map(value => `"${value}"`).join(',');
312-
regexpAllValue = `(${values.join('|')})`;
313-
} else if (!allValue) {
314-
allValue = '*';
315-
regexpAllValue = '.*';
316-
}
317-
318-
queryExpr = queryExpr.replaceAll(`~"$${variableName}"`, `~"${regexpAllValue}"`);
319-
queryExpr = queryExpr.replaceAll(`$${variableName}`, allValue);
320-
return queryExpr;
321-
}
322-
323292
replaceOperatorsToInForMultiQueryVariables(expr: string) {
324293
const variables = this.templateSrv.getVariables();
325294
const fieldValuesVariables = variables.filter(v => v.type === 'query' && v.query.type === 'fieldValue' && v.multi || this.isAllOption(v)) as QueryVariableModel[];
@@ -328,7 +297,7 @@ export class VictoriaLogsDatasource
328297
result = removeDoubleQuotesAroundVar(result, variable.name);
329298
result = replaceOperatorWithIn(result, variable.name);
330299
if (this.isAllOption(variable)) {
331-
result = this.replaceAllOption(result, variable);
300+
result = replaceAllOptionInQuery(result, variable);
332301
}
333302
}
334303
return result;

‎src/modifyQuery.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,9 +109,9 @@ export const logsSortOrders = {
109109
desc: "Descending"
110110
};
111111

112-
export const addSortPipeToQuery = ({ expr, queryType }: Query, sortDirection: string) => {
113-
// if a query is not 'Raw logs' do not add sort pipe
114-
if (queryType !== QueryType.Instant) {
112+
export const addSortPipeToQuery = ({ expr, queryType }: Query, sortDirection: string, isLiveStreaming = false) => {
113+
// if a query is not 'Raw logs' or is a live stream, don't add sort pipe
114+
if (queryType !== QueryType.Instant || isLiveStreaming) {
115115
return expr;
116116
}
117117
const exprContainsSort = /\|\s*sort\s*by\s*\(/i.test(expr); // checks for existing sort pipe `sort by (`

‎src/utils/replaceAllOptionInQuery.test.ts‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
import { QueryVariableModel } from "@grafana/data";
2+
3+
import { replaceAllOptionInQuery } from "./replaceAllOptionInQuery";
4+
5+
describe('replaceAllOptionInQuery', () => {
6+
it('should replace variables with custom allValue', () => {
7+
const queryExpr = 'namespace: $namespace';
8+
const variable: QueryVariableModel = {
9+
name: 'namespace',
10+
allValue: 'all_namespaces',
11+
options: [],
12+
} as any;
13+
14+
const result = replaceAllOptionInQuery(queryExpr, variable);
15+
expect(result).toBe('namespace: all_namespaces');
16+
});
17+
18+
it('should use options list if allValue is not provided and allowCustomValue', () => {
19+
const queryExpr = 'namespace:~"$namespace"';
20+
const variable = {
21+
name: 'namespace',
22+
allValue: null,
23+
allowCustomValue: true,
24+
options: [{ value: 'ns1' }, { value: 'ns2' }],
25+
} as any;
26+
27+
const result = replaceAllOptionInQuery(queryExpr, variable);
28+
expect(result).toBe('namespace:~"(\"ns1\"|\"ns2\")"');
29+
});
30+
31+
it('should use options list if allValue is not provided and query defined', () => {
32+
const queryExpr = 'namespace:~"$namespace"';
33+
const variable = {
34+
name: 'namespace',
35+
allValue: null,
36+
options: [{ value: 'ns1' }, { value: 'ns2' }],
37+
query: {
38+
query: 'filter'
39+
}
40+
} as any;
41+
42+
const result = replaceAllOptionInQuery(queryExpr, variable);
43+
expect(result).toBe('namespace:~"(\"ns1\"|\"ns2\")"');
44+
});
45+
46+
it('should use default wildcard if allValue and options are missing', () => {
47+
const queryExpr = 'namespace:~"$namespace"';
48+
const variable = {
49+
name: 'namespace',
50+
allValue: null,
51+
options: [],
52+
} as any;
53+
54+
const result = replaceAllOptionInQuery(queryExpr, variable);
55+
expect(result).toBe('namespace:~".*"');
56+
});
57+
58+
it('should replace variables when regex and allowCustomValue are enabled', () => {
59+
const queryExpr = 'namespace:~"$namespace"';
60+
const variable = {
61+
name: 'namespace',
62+
allValue: null,
63+
allowCustomValue: true,
64+
regex: true,
65+
options: [{ value: 'ns1' }, { value: 'ns2' }],
66+
} as any;
67+
68+
const result = replaceAllOptionInQuery(queryExpr, variable);
69+
expect(result).toBe('namespace:~"(\"ns1\"|\"ns2\")"');
70+
});
71+
72+
it('should replace multiple occurrences of the same variable', () => {
73+
const queryExpr = 'namespace:$namespace AND pod:$namespace';
74+
const variable = {
75+
name: 'namespace',
76+
allValue: 'all_namespaces',
77+
options: [],
78+
} as any;
79+
80+
const result = replaceAllOptionInQuery(queryExpr, variable);
81+
expect(result).toBe('namespace:all_namespaces AND pod:all_namespaces');
82+
});
83+
84+
it('should change .* on * for non regexp operator', () => {
85+
const queryExpr = 'namespace:~"$namespace" pod:in($namespace)';
86+
const variable = {
87+
name: 'namespace',
88+
allValue: '.*',
89+
options: [],
90+
} as any;
91+
92+
const result = replaceAllOptionInQuery(queryExpr, variable);
93+
expect(result).toBe('namespace:~".*" pod:in(*)');
94+
});
95+
96+
it('should change * on .* for regexp operator', () => {
97+
const queryExpr = 'namespace:~"$namespace" pod:in($namespace)';
98+
const variable = {
99+
name: 'namespace',
100+
allValue: '*',
101+
options: [],
102+
} as any;
103+
104+
const result = replaceAllOptionInQuery(queryExpr, variable);
105+
expect(result).toBe('namespace:~".*" pod:in(*)');
106+
});
107+
108+
it('should change only foo as word', () => {
109+
const queryExpr = 'namespace:in($foo) pod:in($foobar)';
110+
const variable = {
111+
name: 'foo',
112+
allValue: '*',
113+
options: [],
114+
} as any;
115+
116+
const result = replaceAllOptionInQuery(queryExpr, variable);
117+
expect(result).toBe('namespace:in(*) pod:in($foobar)');
118+
});
119+
120+
it('should change only foo as word in regexp', () => {
121+
const queryExpr = 'namespace:~"$foo" pod:~"$foobar"';
122+
const variable = {
123+
name: 'foo',
124+
allValue: '*',
125+
options: [],
126+
} as any;
127+
128+
const result = replaceAllOptionInQuery(queryExpr, variable);
129+
expect(result).toBe('namespace:~".*" pod:~"$foobar"');
130+
});
131+
});

‎src/utils/replaceAllOptionInQuery.ts‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import { QueryVariableModel } from "@grafana/data";
2+
3+
const DEFAULT_ALL_VALUE = '*';
4+
const DEFAULT_REGEXP_ALL_VALUE = '.*';
5+
6+
/**
7+
* Replaces all occurrences of a variable in the query expression with the following priority:
8+
* 1. Custom All Value
9+
* 2. list of variables (if query defined)
10+
* 3. default '*' and '.*' for regex
11+
*
12+
* @param {string} queryExpr - The query expression where the variable needs to be replaced.
13+
* @param {QueryVariableModel} variable - The variable model containing the variable name, options, and allValue.
14+
* @return {string} - The modified query expression with the variable replaced by its corresponding values or patterns.
15+
*/
16+
export function replaceAllOptionInQuery(queryExpr: string, variable: QueryVariableModel): string {
17+
const variableName = variable.name;
18+
const allValue = normalizeAllValue(computeAllValue(variable));
19+
const regexpAllValue = normalizeRegexpAllValue(computeRegexpAllValue(variable));
20+
21+
const regexpPattern = new RegExp(`~"\\$${variableName}\\b"`, 'g');
22+
const variablePattern = new RegExp(`\\$${variableName}\\b`, 'g');
23+
24+
queryExpr = queryExpr.replace(regexpPattern, `~"${regexpAllValue}"`);
25+
queryExpr = queryExpr.replace(variablePattern, allValue);
26+
return queryExpr;
27+
}
28+
29+
function hasModifiedAllOption(variable: QueryVariableModel): boolean {
30+
return Boolean(variable.query?.query || variable.allowCustomValue || variable.regex);
31+
}
32+
33+
function extractOptionValues(variable: QueryVariableModel): string[] {
34+
return variable.options.map(option => option.value).flat(1);
35+
}
36+
37+
function computeAllValue(variable: QueryVariableModel): string {
38+
if (variable.allValue) {
39+
return variable.allValue;
40+
}
41+
42+
if (hasModifiedAllOption(variable)) {
43+
const values = extractOptionValues(variable);
44+
return values.map(value => `"${value}"`).join(',');
45+
}
46+
47+
return DEFAULT_ALL_VALUE;
48+
}
49+
50+
function computeRegexpAllValue(variable: QueryVariableModel): string {
51+
if (variable.allValue) {
52+
return variable.allValue;
53+
}
54+
55+
if (hasModifiedAllOption(variable)) {
56+
const values = extractOptionValues(variable);
57+
return `(${values.map(value => `\"${value}\"`).join('|')})`;
58+
}
59+
60+
return DEFAULT_REGEXP_ALL_VALUE;
61+
}
62+
63+
// if allValue is default '.*', convert to '*' for non-regex usage
64+
function normalizeAllValue(allValue: string): string {
65+
return allValue === DEFAULT_REGEXP_ALL_VALUE ? DEFAULT_ALL_VALUE : allValue;
66+
}
67+
68+
// if allValue is default '*', convert to '.*' for regex usage
69+
function normalizeRegexpAllValue(regexpAllValue: string): string {
70+
return regexpAllValue === DEFAULT_ALL_VALUE ? DEFAULT_REGEXP_ALL_VALUE : regexpAllValue;
71+
}

0 commit comments

Comments
 (0)