Skip to content

Commit 5628f94

Browse files
SlachSlach
authored
Merge pull request #778 from Altinity/issue-777
Add nullify sparse logic
2 parents 4f48727 + cf64af9 commit 5628f94

File tree

13 files changed

+95
-30
lines changed

13 files changed

+95
-30
lines changed

‎src/datasource/datasource.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ export class CHDataSource
8484
},
8585
defaultDateTimeType: instanceSettings.jsonData.defaultDateTimeType,
8686
contextWindowSize: instanceSettings.jsonData.contextWindowSize,
87+
nullifySparse: instanceSettings.jsonData.nullifySparse,
8788
};
8889
}
8990

@@ -466,7 +467,7 @@ export class CHDataSource
466467

467468
result = [resultContent]
468469
} else {
469-
_.each(sqlSeries.toTimeSeries(target.extrapolate), (data) => {
470+
_.each(sqlSeries.toTimeSeries(target.extrapolate, target.nullifySparse), (data) => {
470471
result.push(data);
471472
});
472473
}

‎src/datasource/sql-series/sql_series.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,9 +143,9 @@ export default class SqlSeries {
143143
return toTable(self);
144144
};
145145

146-
toTimeSeries = (extrapolate = true): any => {
146+
toTimeSeries = (extrapolate = true, nullifySparse = false): any => {
147147
let self = this;
148-
return toTimeSeries(extrapolate, self);
148+
return toTimeSeries(extrapolate, nullifySparse, self);
149149
};
150150

151151
toTraces = (): any => {

‎src/datasource/sql-series/toTimeSeries.ts

Lines changed: 22 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -68,24 +68,27 @@ const extrapolateDataPoints = (datapoints: any, self) => {
6868
return datapoints;
6969
};
7070

71-
const _pushDatapoint = (metrics: any, timestamp: number, key: string, value: number) => {
71+
const _pushDatapoint = (metrics: any, timestamp: number, key: string, value: number, nullifySparse: boolean) => {
7272
if (!metrics[key]) {
7373
metrics[key] = [];
74-
/* Fill null values for each new series */
75-
for (let seriesName in metrics) {
76-
metrics[seriesName].forEach((v: any) => {
77-
if (v[1] < timestamp) {
78-
metrics[key].push([null, v[1]]);
79-
}
80-
});
81-
break;
74+
75+
/* Fill null values for each new series only if nullifySparse is true */
76+
if (nullifySparse) {
77+
for (let seriesName in metrics) {
78+
metrics[seriesName].forEach((v: any) => {
79+
if (v[1] < timestamp) {
80+
metrics[key].push([null, v[1]]);
81+
}
82+
});
83+
break;
84+
}
8285
}
8386
}
8487

8588
metrics[key].push([_formatValue(value), timestamp]);
8689
};
8790

88-
export const toTimeSeries = (extrapolate = true, self): any => {
91+
export const toTimeSeries = (extrapolate = true, nullifySparse = false, self): any => {
8992
let timeSeries: any[] = [];
9093
if (self.series.length === 0) {
9194
return timeSeries;
@@ -125,11 +128,13 @@ export const toTimeSeries = (extrapolate = true, self): any => {
125128
/* Make sure all series end with a value or nil for current timestamp
126129
* to render discontinuous timeseries properly. */
127130
if (lastTimeStamp < t) {
128-
each(metrics, function (dataPoints, seriesName) {
129-
if (dataPoints[dataPoints.length - 1][1] < lastTimeStamp) {
130-
dataPoints.push([null, lastTimeStamp]);
131-
}
132-
});
131+
if (nullifySparse) {
132+
each(metrics, function (dataPoints, seriesName) {
133+
if (dataPoints[dataPoints.length - 1][1] < lastTimeStamp) {
134+
dataPoints.push([null, lastTimeStamp]);
135+
}
136+
});
137+
}
133138
lastTimeStamp = t;
134139
}
135140
/* For each metric-value pair in row, construct a datapoint */
@@ -150,10 +155,10 @@ export const toTimeSeries = (extrapolate = true, self): any => {
150155
if (isArray(val)) {
151156
/* Expand groupArray into multiple timeseries */
152157
each(val, function (arr) {
153-
_pushDatapoint(metrics, t, arr[0], arr[1]);
158+
_pushDatapoint(metrics, t, arr[0], arr[1], nullifySparse);
154159
});
155160
} else {
156-
_pushDatapoint(metrics, t, key, val);
161+
_pushDatapoint(metrics, t, key, val, nullifySparse);
157162
}
158163
});
159164
});

‎src/spec/datasource.jest.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ describe('clickhouse sql series:', () => {
109109
meta: response.meta,
110110
table: '',
111111
});
112-
let timeSeries = sqlSeries.toTimeSeries();
112+
let timeSeries = sqlSeries.toTimeSeries(false, true);
113113

114114
it('expects four results', () => {
115115
expect(size(timeSeries)).toBe(4);

‎src/spec/sql_series_specs.jest.ts

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,7 @@ describe('sql-series. toTimeSeries unit tests', () => {
301301
};
302302

303303
it('should return an empty array when there are no series', () => {
304-
const result = toTimeSeries(true, selfMock);
304+
const result = toTimeSeries(true, true, selfMock);
305305
expect(result).toEqual([]);
306306
});
307307

@@ -313,7 +313,7 @@ describe('sql-series. toTimeSeries unit tests', () => {
313313
];
314314
selfMock.keys = [];
315315

316-
const result = toTimeSeries(true, selfMock);
316+
const result = toTimeSeries(true, true, selfMock);
317317
expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000]}, {"config": {"links": []}, "name": "value", "values": [10]}], "length": 1, "refId": undefined}]);
318318
});
319319

@@ -328,22 +328,21 @@ describe('sql-series. toTimeSeries unit tests', () => {
328328
];
329329
selfMock.keys = [];
330330

331-
const result = toTimeSeries(true, selfMock);
331+
const result = toTimeSeries(true, true, selfMock);
332332
expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000, 2000]}, {"config": {"links": []}, "name": "value", "values": [10, 20]}], "length": 2, "refId": undefined}]);
333333
});
334334

335335
it('should extrapolate data points when required', () => {
336336
let selfMock = {"from": 0, "keys": [], "meta": [{"name": "time", "type": "UInt32"}, {"name": "value", "type": "UInt64"}], "series": [{"time": 1736332351828, "value": 32}, {"time": 1736332336828, "value": 34}, {"time": 1736332321828, "value": 36}, {"time": 1736332306828, "value": 38}, {"time": 1736332291828, "value": 40}, {"time": 1736332276828, "value": 42}, {"time": 1736332261828, "value": 44}, {"time": 1736332246828, "value": 46}, {"time": 1736332231828, "value": 48}, {"time": 1736332216828, "value": 50}], "tillNow": true, "to": 1000}
337-
const result = toTimeSeries(true, selfMock);
337+
const result = toTimeSeries(true, true, selfMock);
338338
expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1736332351828, 1736332336828, 1736332321828, 1736332306828, 1736332291828, 1736332276828, 1736332261828, 1736332246828, 1736332231828, 1736332216828]}, {"config": {"links": []}, "name": "value", "values": [32, 34, 36, 38, 40, 42, 44, 46, 48, 48.2]}], "length": 10, "refId": undefined}]);
339339

340340
selfMock = {"from": 0, "keys": [], "meta": [{"name": "time", "type": "UInt32"}, {"name": "value", "type": "UInt64"}], "series": [{"time": 1736332580592, "value": 52}, {"time": 1736332550592, "value": 54}, {"time": 1736332520592, "value": 56}], "tillNow": true, "to": 1000}
341-
const resultNonExtrapolated = toTimeSeries(true, selfMock);
341+
const resultNonExtrapolated = toTimeSeries(true, true, selfMock);
342342
expect(resultNonExtrapolated).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1736332580592, 1736332550592, 1736332520592]}, {"config": {"links": []}, "name": "value", "values": [52, 54, 56]}], "length": 3, "refId": undefined}]);
343-
344343
});
345344

346-
it('should handle composite keys correctly', () => {
345+
it('should handle composite keys correctly with nullifySparse=true', () => {
347346
selfMock.series = [
348347
{ time: 1000, category: 'A', value: 10 },
349348
{ time: 2000, category: 'B', value: 20 },
@@ -356,10 +355,11 @@ describe('sql-series. toTimeSeries unit tests', () => {
356355
selfMock.keys = ['category'];
357356
selfMock.tillNow = false;
358357

359-
const result = toTimeSeries(true, selfMock);
358+
const result = toTimeSeries(true, true, selfMock);
360359
expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000, 1000]}, {"config": {"links": []}, "name": "A", "values": [1000, 10]}], "length": 2, "refId": undefined}, {"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000, 1000, 2000, 2000]}, {"config": {"links": []}, "name": "B", "values": [null, null, 2000, 20]}], "length": 4, "refId": undefined}])
361360
});
362361

362+
363363
it('should handle null values correctly', () => {
364364
selfMock.series = [
365365
{ time: 1000, value: null },
@@ -371,7 +371,7 @@ describe('sql-series. toTimeSeries unit tests', () => {
371371
];
372372
selfMock.keys = [];
373373

374-
const result = toTimeSeries(false, selfMock);
374+
const result = toTimeSeries(false, true, selfMock);
375375
expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000, 2000]}, {"config": {"links": []}, "name": "value", "values": [null, 20]}], "length": 2, "refId": undefined}]);
376376
});
377377
});

‎src/types/types.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ export interface CHQuery extends DataQuery {
4848

4949
skip_comments?: boolean;
5050
add_metadata?: boolean;
51+
nullifySparse?: boolean;
5152

5253
round?: string;
5354
intervalFactor?: number;
@@ -88,6 +89,7 @@ export interface CHDataSourceOptions extends DataSourceJsonData {
8889
adHocHideTableNames?: boolean;
8990
contextWindowSize?: string;
9091
useWindowFuncForMacros?: boolean;
92+
nullifySparse?: boolean;
9193
}
9294

9395
/**
@@ -105,4 +107,5 @@ export const DEFAULT_QUERY: CHQuery = {
105107
showHelp: false,
106108
showFormattedSQL: false,
107109
datasourceMode: DatasourceMode.Datasource,
110+
nullifySparse: false,
108111
};

‎src/views/ConfigEditor/FormParts/DefaultValues/DefaultValues.tsx

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,14 @@ export const DefaultValues = ({ jsonData, newOptions, onSwitchToggle, onFieldCha
338338
onChange={(e) => onSwitchToggle('useWindowFuncForMacros', e.currentTarget.checked)}
339339
/>
340340
</InlineField>
341+
<InlineField label="Nullify sparse categories" labelWidth={32} style={{ marginLeft: '30px' }}>
342+
<InlineSwitch
343+
id="nullifySparse"
344+
data-testid="nullify-sparse-switch"
345+
value={jsonData.nullifySparse ?? false}
346+
onChange={(e) => onSwitchToggle('nullifySparse', e.currentTarget.checked)}
347+
/>
348+
</InlineField>
341349
</>
342350
)}
343351
</div>

‎src/views/QueryEditor/components/QueryHeader/helpers/findDifferences.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ export function findDifferences(query: CHQuery, datasource: CHDataSource) {
9191
fieldName: 'useWindowFuncForMacros',
9292
});
9393
}
94+
if (query.nullifySparse !== defaultValues.nullifySparse) {
95+
differences.push({
96+
key: 'Nullify sparse categories',
97+
original: query.nullifySparse?.toString() || 'false',
98+
updated: defaultValues.nullifySparse?.toString() || 'false',
99+
fieldName: 'nullifySparse',
100+
});
101+
}
94102
}
95103

96104
return differences;

‎src/views/QueryEditor/components/QueryTextEditor/QueryTextEditor.tsx

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
RoundInput,
1414
MetadataSwitch,
1515
SkipCommentsSwitch,
16+
NullifySparseSwitch,
1617
UseWindowFunctionSwitch,
1718
FormatAsSelect,
1819
ContextWindowSizeSelect,
@@ -80,6 +81,10 @@ export const QueryTextEditor: React.FC<QueryTextEditorProps> = ({
8081
query={query}
8182
onChange={() => handlers.handleToggleField('skip_comments')}
8283
/>
84+
<NullifySparseSwitch
85+
query={query}
86+
onChange={() => handlers.handleToggleField('nullifySparse')}
87+
/>
8388
<UseWindowFunctionSwitch
8489
query={query}
8590
onChange={() => handlers.handleToggleField('useWindowFuncForMacros')}

‎src/views/QueryEditor/components/QueryTextEditor/components/Switches/NullifySparseSwitch.tsx

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import React from 'react';
2+
import { InlineField, InlineLabel, InlineSwitch } from '@grafana/ui';
3+
import { SwitchProps } from '../../types';
4+
5+
export const NullifySparseSwitch: React.FC<SwitchProps> = ({ query, onChange }) => (
6+
<InlineField
7+
label={
8+
<InlineLabel width={18} tooltip="Replace sparse categories with NULL values">
9+
Nullify Sparse
10+
</InlineLabel>
11+
}
12+
style={{ height: '100%' }}
13+
>
14+
<InlineSwitch
15+
data-testid="nullify-sparse-switch"
16+
width="auto"
17+
value={query.nullifySparse}
18+
onChange={onChange}
19+
transparent
20+
/>
21+
</InlineField>
22+
);

0 commit comments

Comments
 (0)