Skip to content

Commit 5628f94

Browse files
SlachSlach
authored
Merge pull request #778 from Altinity/issue-777
Add nullify sparse logic
2 parents 4f48727 + cf64af9 commit 5628f94

File tree

13 files changed

+95
-30
lines changed

13 files changed

+95
-30
lines changed

‎src/datasource/datasource.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ export class CHDataSource
8484
},
8585
defaultDateTimeType: instanceSettings.jsonData.defaultDateTimeType,
8686
contextWindowSize: instanceSettings.jsonData.contextWindowSize,
87+
nullifySparse: instanceSettings.jsonData.nullifySparse,
8788
};
8889
}
8990

@@ -466,7 +467,7 @@ export class CHDataSource
466467

467468
result = [resultContent]
468469
} else {
469-
_.each(sqlSeries.toTimeSeries(target.extrapolate), (data) => {
470+
_.each(sqlSeries.toTimeSeries(target.extrapolate, target.nullifySparse), (data) => {
470471
result.push(data);
471472
});
472473
}

‎src/datasource/sql-series/sql_series.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -143,9 +143,9 @@ export default class SqlSeries {
143143
return toTable(self);
144144
};
145145

146-
toTimeSeries = (extrapolate = true): any => {
146+
toTimeSeries = (extrapolate = true, nullifySparse = false): any => {
147147
let self = this;
148-
return toTimeSeries(extrapolate, self);
148+
return toTimeSeries(extrapolate, nullifySparse, self);
149149
};
150150

151151
toTraces = (): any => {

‎src/datasource/sql-series/toTimeSeries.ts

+22-17
Original file line numberDiff line numberDiff line change
@@ -68,24 +68,27 @@ const extrapolateDataPoints = (datapoints: any, self) => {
6868
return datapoints;
6969
};
7070

71-
const _pushDatapoint = (metrics: any, timestamp: number, key: string, value: number) => {
71+
const _pushDatapoint = (metrics: any, timestamp: number, key: string, value: number, nullifySparse: boolean) => {
7272
if (!metrics[key]) {
7373
metrics[key] = [];
74-
/* Fill null values for each new series */
75-
for (let seriesName in metrics) {
76-
metrics[seriesName].forEach((v: any) => {
77-
if (v[1] < timestamp) {
78-
metrics[key].push([null, v[1]]);
79-
}
80-
});
81-
break;
74+
75+
/* Fill null values for each new series only if nullifySparse is true */
76+
if (nullifySparse) {
77+
for (let seriesName in metrics) {
78+
metrics[seriesName].forEach((v: any) => {
79+
if (v[1] < timestamp) {
80+
metrics[key].push([null, v[1]]);
81+
}
82+
});
83+
break;
84+
}
8285
}
8386
}
8487

8588
metrics[key].push([_formatValue(value), timestamp]);
8689
};
8790

88-
export const toTimeSeries = (extrapolate = true, self): any => {
91+
export const toTimeSeries = (extrapolate = true, nullifySparse = false, self): any => {
8992
let timeSeries: any[] = [];
9093
if (self.series.length === 0) {
9194
return timeSeries;
@@ -125,11 +128,13 @@ export const toTimeSeries = (extrapolate = true, self): any => {
125128
/* Make sure all series end with a value or nil for current timestamp
126129
* to render discontinuous timeseries properly. */
127130
if (lastTimeStamp < t) {
128-
each(metrics, function (dataPoints, seriesName) {
129-
if (dataPoints[dataPoints.length - 1][1] < lastTimeStamp) {
130-
dataPoints.push([null, lastTimeStamp]);
131-
}
132-
});
131+
if (nullifySparse) {
132+
each(metrics, function (dataPoints, seriesName) {
133+
if (dataPoints[dataPoints.length - 1][1] < lastTimeStamp) {
134+
dataPoints.push([null, lastTimeStamp]);
135+
}
136+
});
137+
}
133138
lastTimeStamp = t;
134139
}
135140
/* For each metric-value pair in row, construct a datapoint */
@@ -150,10 +155,10 @@ export const toTimeSeries = (extrapolate = true, self): any => {
150155
if (isArray(val)) {
151156
/* Expand groupArray into multiple timeseries */
152157
each(val, function (arr) {
153-
_pushDatapoint(metrics, t, arr[0], arr[1]);
158+
_pushDatapoint(metrics, t, arr[0], arr[1], nullifySparse);
154159
});
155160
} else {
156-
_pushDatapoint(metrics, t, key, val);
161+
_pushDatapoint(metrics, t, key, val, nullifySparse);
157162
}
158163
});
159164
});

‎src/spec/datasource.jest.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ describe('clickhouse sql series:', () => {
109109
meta: response.meta,
110110
table: '',
111111
});
112-
let timeSeries = sqlSeries.toTimeSeries();
112+
let timeSeries = sqlSeries.toTimeSeries(false, true);
113113

114114
it('expects four results', () => {
115115
expect(size(timeSeries)).toBe(4);

‎src/spec/sql_series_specs.jest.ts

+9-9
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,7 @@ describe('sql-series. toTimeSeries unit tests', () => {
301301
};
302302

303303
it('should return an empty array when there are no series', () => {
304-
const result = toTimeSeries(true, selfMock);
304+
const result = toTimeSeries(true, true, selfMock);
305305
expect(result).toEqual([]);
306306
});
307307

@@ -313,7 +313,7 @@ describe('sql-series. toTimeSeries unit tests', () => {
313313
];
314314
selfMock.keys = [];
315315

316-
const result = toTimeSeries(true, selfMock);
316+
const result = toTimeSeries(true, true, selfMock);
317317
expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000]}, {"config": {"links": []}, "name": "value", "values": [10]}], "length": 1, "refId": undefined}]);
318318
});
319319

@@ -328,22 +328,21 @@ describe('sql-series. toTimeSeries unit tests', () => {
328328
];
329329
selfMock.keys = [];
330330

331-
const result = toTimeSeries(true, selfMock);
331+
const result = toTimeSeries(true, true, selfMock);
332332
expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000, 2000]}, {"config": {"links": []}, "name": "value", "values": [10, 20]}], "length": 2, "refId": undefined}]);
333333
});
334334

335335
it('should extrapolate data points when required', () => {
336336
let selfMock = {"from": 0, "keys": [], "meta": [{"name": "time", "type": "UInt32"}, {"name": "value", "type": "UInt64"}], "series": [{"time": 1736332351828, "value": 32}, {"time": 1736332336828, "value": 34}, {"time": 1736332321828, "value": 36}, {"time": 1736332306828, "value": 38}, {"time": 1736332291828, "value": 40}, {"time": 1736332276828, "value": 42}, {"time": 1736332261828, "value": 44}, {"time": 1736332246828, "value": 46}, {"time": 1736332231828, "value": 48}, {"time": 1736332216828, "value": 50}], "tillNow": true, "to": 1000}
337-
const result = toTimeSeries(true, selfMock);
337+
const result = toTimeSeries(true, true, selfMock);
338338
expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1736332351828, 1736332336828, 1736332321828, 1736332306828, 1736332291828, 1736332276828, 1736332261828, 1736332246828, 1736332231828, 1736332216828]}, {"config": {"links": []}, "name": "value", "values": [32, 34, 36, 38, 40, 42, 44, 46, 48, 48.2]}], "length": 10, "refId": undefined}]);
339339

340340
selfMock = {"from": 0, "keys": [], "meta": [{"name": "time", "type": "UInt32"}, {"name": "value", "type": "UInt64"}], "series": [{"time": 1736332580592, "value": 52}, {"time": 1736332550592, "value": 54}, {"time": 1736332520592, "value": 56}], "tillNow": true, "to": 1000}
341-
const resultNonExtrapolated = toTimeSeries(true, selfMock);
341+
const resultNonExtrapolated = toTimeSeries(true, true, selfMock);
342342
expect(resultNonExtrapolated).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1736332580592, 1736332550592, 1736332520592]}, {"config": {"links": []}, "name": "value", "values": [52, 54, 56]}], "length": 3, "refId": undefined}]);
343-
344343
});
345344

346-
it('should handle composite keys correctly', () => {
345+
it('should handle composite keys correctly with nullifySparse=true', () => {
347346
selfMock.series = [
348347
{ time: 1000, category: 'A', value: 10 },
349348
{ time: 2000, category: 'B', value: 20 },
@@ -356,10 +355,11 @@ describe('sql-series. toTimeSeries unit tests', () => {
356355
selfMock.keys = ['category'];
357356
selfMock.tillNow = false;
358357

359-
const result = toTimeSeries(true, selfMock);
358+
const result = toTimeSeries(true, true, selfMock);
360359
expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000, 1000]}, {"config": {"links": []}, "name": "A", "values": [1000, 10]}], "length": 2, "refId": undefined}, {"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000, 1000, 2000, 2000]}, {"config": {"links": []}, "name": "B", "values": [null, null, 2000, 20]}], "length": 4, "refId": undefined}])
361360
});
362361

362+
363363
it('should handle null values correctly', () => {
364364
selfMock.series = [
365365
{ time: 1000, value: null },
@@ -371,7 +371,7 @@ describe('sql-series. toTimeSeries unit tests', () => {
371371
];
372372
selfMock.keys = [];
373373

374-
const result = toTimeSeries(false, selfMock);
374+
const result = toTimeSeries(false, true, selfMock);
375375
expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000, 2000]}, {"config": {"links": []}, "name": "value", "values": [null, 20]}], "length": 2, "refId": undefined}]);
376376
});
377377
});

‎src/types/types.ts

+3
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ export interface CHQuery extends DataQuery {
4848

4949
skip_comments?: boolean;
5050
add_metadata?: boolean;
51+
nullifySparse?: boolean;
5152

5253
round?: string;
5354
intervalFactor?: number;
@@ -88,6 +89,7 @@ export interface CHDataSourceOptions extends DataSourceJsonData {
8889
adHocHideTableNames?: boolean;
8990
contextWindowSize?: string;
9091
useWindowFuncForMacros?: boolean;
92+
nullifySparse?: boolean;
9193
}
9294

9395
/**
@@ -105,4 +107,5 @@ export const DEFAULT_QUERY: CHQuery = {
105107
showHelp: false,
106108
showFormattedSQL: false,
107109
datasourceMode: DatasourceMode.Datasource,
110+
nullifySparse: false,
108111
};

‎src/views/ConfigEditor/FormParts/DefaultValues/DefaultValues.tsx

+8
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,14 @@ export const DefaultValues = ({ jsonData, newOptions, onSwitchToggle, onFieldCha
338338
onChange={(e) => onSwitchToggle('useWindowFuncForMacros', e.currentTarget.checked)}
339339
/>
340340
</InlineField>
341+
<InlineField label="Nullify sparse categories" labelWidth={32} style={{ marginLeft: '30px' }}>
342+
<InlineSwitch
343+
id="nullifySparse"
344+
data-testid="nullify-sparse-switch"
345+
value={jsonData.nullifySparse ?? false}
346+
onChange={(e) => onSwitchToggle('nullifySparse', e.currentTarget.checked)}
347+
/>
348+
</InlineField>
341349
</>
342350
)}
343351
</div>

‎src/views/QueryEditor/components/QueryHeader/helpers/findDifferences.ts

+8
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ export function findDifferences(query: CHQuery, datasource: CHDataSource) {
9191
fieldName: 'useWindowFuncForMacros',
9292
});
9393
}
94+
if (query.nullifySparse !== defaultValues.nullifySparse) {
95+
differences.push({
96+
key: 'Nullify sparse categories',
97+
original: query.nullifySparse?.toString() || 'false',
98+
updated: defaultValues.nullifySparse?.toString() || 'false',
99+
fieldName: 'nullifySparse',
100+
});
101+
}
94102
}
95103

96104
return differences;

‎src/views/QueryEditor/components/QueryTextEditor/QueryTextEditor.tsx

+5
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
RoundInput,
1414
MetadataSwitch,
1515
SkipCommentsSwitch,
16+
NullifySparseSwitch,
1617
UseWindowFunctionSwitch,
1718
FormatAsSelect,
1819
ContextWindowSizeSelect,
@@ -80,6 +81,10 @@ export const QueryTextEditor: React.FC<QueryTextEditorProps> = ({
8081
query={query}
8182
onChange={() => handlers.handleToggleField('skip_comments')}
8283
/>
84+
<NullifySparseSwitch
85+
query={query}
86+
onChange={() => handlers.handleToggleField('nullifySparse')}
87+
/>
8388
<UseWindowFunctionSwitch
8489
query={query}
8590
onChange={() => handlers.handleToggleField('useWindowFuncForMacros')}

‎src/views/QueryEditor/components/QueryTextEditor/components/Switches/NullifySparseSwitch.tsx

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import React from 'react';
2+
import { InlineField, InlineLabel, InlineSwitch } from '@grafana/ui';
3+
import { SwitchProps } from '../../types';
4+
5+
export const NullifySparseSwitch: React.FC<SwitchProps> = ({ query, onChange }) => (
6+
<InlineField
7+
label={
8+
<InlineLabel width={18} tooltip="Replace sparse categories with NULL values">
9+
Nullify Sparse
10+
</InlineLabel>
11+
}
12+
style={{ height: '100%' }}
13+
>
14+
<InlineSwitch
15+
data-testid="nullify-sparse-switch"
16+
width="auto"
17+
value={query.nullifySparse}
18+
onChange={onChange}
19+
transparent
20+
/>
21+
</InlineField>
22+
);

‎src/views/QueryEditor/components/QueryTextEditor/components/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ export * from './Selects/FormatAsSelect';
66
export * from './Selects/ResolutionsInput';
77
export * from './Switches/ExtrapolationSwitch';
88
export * from './Switches/MetadataSwitch';
9+
export * from './Switches/NullifySparseSwitch';
910
export * from './Switches/SkipCommentsSwitch';
1011
export * from './Switches/UseWindowFunctionSwitch';
1112
export * from './Toolbar/ToolbarButtons';

‎src/views/QueryEditor/components/QueryTextEditor/types.ts

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ export interface Query {
88
round?: string;
99
add_metadata?: boolean;
1010
skip_comments?: boolean;
11+
nullifySparse?: boolean;
1112
useWindowFuncForMacros?: boolean;
1213
format: string;
1314
contextWindowSize?: string;

‎src/views/QueryEditor/helpers/initializeQueryDefaults.ts

+11
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ export const initializeQueryDefaults = (
1313
extrapolate: query.extrapolate ?? true,
1414
skip_comments: query.skip_comments ?? true,
1515
add_metadata: query.add_metadata ?? true,
16+
nullifySparse: query.nullifySparse ?? false,
1617
useWindowFuncForMacros: query.useWindowFuncForMacros ?? true,
1718
dateTimeType: query.dateTimeType,
1819
round: query.round || DEFAULT_ROUND,
@@ -95,6 +96,11 @@ export const initializeQueryDefaults = (
9596
initializedQuery.contextWindowSize = datasource.defaultValues.contextWindowSize;
9697
}
9798

99+
console.log(datasource.defaultValues.nullifySparse, query.nullifySparse, '------')
100+
if (datasource.defaultValues.nullifySparse !== undefined && query.nullifySparse === undefined) {
101+
initializedQuery.nullifySparse = datasource.defaultValues.nullifySparse;
102+
}
103+
98104
onChange({ ...query, ...initializedQuery, initialized: true });
99105
}
100106

@@ -118,6 +124,7 @@ export const initializeQueryDefaultsForVariables = (
118124
extrapolate: query.extrapolate ?? true,
119125
skip_comments: query.skip_comments ?? true,
120126
add_metadata: query.add_metadata ?? true,
127+
nullifySparse: query.nullifySparse ?? false,
121128
useWindowFuncForMacros: query.useWindowFuncForMacros ?? true,
122129
dateTimeType: query.dateTimeType,
123130
round: query.round || DEFAULT_ROUND,
@@ -200,6 +207,10 @@ export const initializeQueryDefaultsForVariables = (
200207
initializedQuery.contextWindowSize = datasource.defaultValues.contextWindowSize;
201208
}
202209

210+
if (datasource.defaultValues.nullifySparse !== undefined && query.nullifySparse === undefined) {
211+
initializedQuery.nullifySparse = datasource.defaultValues.nullifySparse;
212+
}
213+
203214
onChange({ ...query, ...initializedQuery, initialized: true });
204215
}
205216

0 commit comments

Comments
 (0)