Altinity · Slach · Apr 24, 2025 · Apr 22, 2025
diff --git a/src/datasource/datasource.ts b/src/datasource/datasource.ts
@@ -84,6 +84,7 @@ export class CHDataSource
         },
         defaultDateTimeType: instanceSettings.jsonData.defaultDateTimeType,
         contextWindowSize: instanceSettings.jsonData.contextWindowSize,
+        nullifySparse: instanceSettings.jsonData.nullifySparse,
       };
     }
 
@@ -466,7 +467,7 @@ export class CHDataSource
 
             result = [resultContent]
           } else {
-            _.each(sqlSeries.toTimeSeries(target.extrapolate), (data) => {
+            _.each(sqlSeries.toTimeSeries(target.extrapolate, target.nullifySparse), (data) => {
               result.push(data);
             });
           }

diff --git a/src/datasource/sql-series/sql_series.ts b/src/datasource/sql-series/sql_series.ts
@@ -143,9 +143,9 @@ export default class SqlSeries {
     return toTable(self);
   };
 
-  toTimeSeries = (extrapolate = true): any => {
+  toTimeSeries = (extrapolate = true, nullifySparse = false): any => {
     let self = this;
-    return toTimeSeries(extrapolate, self);
+    return toTimeSeries(extrapolate, nullifySparse, self);
   };
 
   toTraces = (): any => {

diff --git a/src/datasource/sql-series/toTimeSeries.ts b/src/datasource/sql-series/toTimeSeries.ts
@@ -68,24 +68,27 @@ const extrapolateDataPoints = (datapoints: any, self) => {
   return datapoints;
 };
 
-const _pushDatapoint = (metrics: any, timestamp: number, key: string, value: number) => {
+const _pushDatapoint = (metrics: any, timestamp: number, key: string, value: number, nullifySparse: boolean) => {
   if (!metrics[key]) {
     metrics[key] = [];
-    /* Fill null values for each new series */
-    for (let seriesName in metrics) {
-      metrics[seriesName].forEach((v: any) => {
-        if (v[1] < timestamp) {
-          metrics[key].push([null, v[1]]);
-        }
-      });
-      break;
+
+    /* Fill null values for each new series only if nullifySparse is true */
+    if (nullifySparse) {
+      for (let seriesName in metrics) {
+        metrics[seriesName].forEach((v: any) => {
+          if (v[1] < timestamp) {
+            metrics[key].push([null, v[1]]);
+          }
+        });
+        break;
+      }
     }
   }
 
   metrics[key].push([_formatValue(value), timestamp]);
 };
 
-export const toTimeSeries = (extrapolate = true, self): any => {
+export const toTimeSeries = (extrapolate = true, nullifySparse = false, self): any => {
   let timeSeries: any[] = [];
   if (self.series.length === 0) {
     return timeSeries;
@@ -125,11 +128,13 @@ export const toTimeSeries = (extrapolate = true, self): any => {
     /* Make sure all series end with a value or nil for current timestamp
      * to render discontinuous timeseries properly. */
     if (lastTimeStamp < t) {
-      each(metrics, function (dataPoints, seriesName) {
-        if (dataPoints[dataPoints.length - 1][1] < lastTimeStamp) {
-          dataPoints.push([null, lastTimeStamp]);
-        }
-      });
+      if (nullifySparse) {
+        each(metrics, function (dataPoints, seriesName) {
+          if (dataPoints[dataPoints.length - 1][1] < lastTimeStamp) {
+            dataPoints.push([null, lastTimeStamp]);
+          }
+        });
+      }
       lastTimeStamp = t;
     }
     /* For each metric-value pair in row, construct a datapoint */
@@ -150,10 +155,10 @@ export const toTimeSeries = (extrapolate = true, self): any => {
       if (isArray(val)) {
         /* Expand groupArray into multiple timeseries */
         each(val, function (arr) {
-          _pushDatapoint(metrics, t, arr[0], arr[1]);
+          _pushDatapoint(metrics, t, arr[0], arr[1], nullifySparse);
         });
       } else {
-        _pushDatapoint(metrics, t, key, val);
+        _pushDatapoint(metrics, t, key, val, nullifySparse);
       }
     });
   });

diff --git a/src/spec/datasource.jest.ts b/src/spec/datasource.jest.ts
@@ -109,7 +109,7 @@ describe('clickhouse sql series:', () => {
       meta: response.meta,
       table: '',
     });
-    let timeSeries = sqlSeries.toTimeSeries();
+    let timeSeries = sqlSeries.toTimeSeries(false, true);
 
     it('expects four results', () => {
       expect(size(timeSeries)).toBe(4);

diff --git a/src/spec/sql_series_specs.jest.ts b/src/spec/sql_series_specs.jest.ts
@@ -301,7 +301,7 @@ describe('sql-series. toTimeSeries unit tests', () => {
   };
 
   it('should return an empty array when there are no series', () => {
-    const result = toTimeSeries(true, selfMock);
+    const result = toTimeSeries(true, true, selfMock);
     expect(result).toEqual([]);
   });
 
@@ -313,7 +313,7 @@ describe('sql-series. toTimeSeries unit tests', () => {
     ];
     selfMock.keys = [];
 
-    const result = toTimeSeries(true, selfMock);
+    const result = toTimeSeries(true, true, selfMock);
     expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000]}, {"config": {"links": []}, "name": "value", "values": [10]}], "length": 1, "refId": undefined}]);
   });
 
@@ -328,22 +328,21 @@ describe('sql-series. toTimeSeries unit tests', () => {
     ];
     selfMock.keys = [];
 
-    const result = toTimeSeries(true, selfMock);
+    const result = toTimeSeries(true, true, selfMock);
     expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000, 2000]}, {"config": {"links": []}, "name": "value", "values": [10, 20]}], "length": 2, "refId": undefined}]);
   });
 
   it('should extrapolate data points when required', () => {
     let selfMock = {"from": 0, "keys": [], "meta": [{"name": "time", "type": "UInt32"}, {"name": "value", "type": "UInt64"}], "series": [{"time": 1736332351828, "value": 32}, {"time": 1736332336828, "value": 34}, {"time": 1736332321828, "value": 36}, {"time": 1736332306828, "value": 38}, {"time": 1736332291828, "value": 40}, {"time": 1736332276828, "value": 42}, {"time": 1736332261828, "value": 44}, {"time": 1736332246828, "value": 46}, {"time": 1736332231828, "value": 48}, {"time": 1736332216828, "value": 50}], "tillNow": true, "to": 1000}
-    const result = toTimeSeries(true, selfMock);
+    const result = toTimeSeries(true, true, selfMock);
     expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1736332351828, 1736332336828, 1736332321828, 1736332306828, 1736332291828, 1736332276828, 1736332261828, 1736332246828, 1736332231828, 1736332216828]}, {"config": {"links": []}, "name": "value", "values": [32, 34, 36, 38, 40, 42, 44, 46, 48, 48.2]}], "length": 10, "refId": undefined}]);
 
     selfMock = {"from": 0, "keys": [], "meta": [{"name": "time", "type": "UInt32"}, {"name": "value", "type": "UInt64"}], "series": [{"time": 1736332580592, "value": 52}, {"time": 1736332550592, "value": 54}, {"time": 1736332520592, "value": 56}], "tillNow": true, "to": 1000}
-    const resultNonExtrapolated = toTimeSeries(true, selfMock);
+    const resultNonExtrapolated = toTimeSeries(true, true, selfMock);
     expect(resultNonExtrapolated).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1736332580592, 1736332550592, 1736332520592]}, {"config": {"links": []}, "name": "value", "values": [52, 54, 56]}], "length": 3, "refId": undefined}]);
-
   });
 
-  it('should handle composite keys correctly', () => {
+  it('should handle composite keys correctly with nullifySparse=true', () => {
     selfMock.series = [
       { time: 1000, category: 'A', value: 10 },
       { time: 2000, category: 'B', value: 20 },
@@ -356,10 +355,11 @@ describe('sql-series. toTimeSeries unit tests', () => {
     selfMock.keys = ['category'];
     selfMock.tillNow = false;
 
-    const result = toTimeSeries(true, selfMock);
+    const result = toTimeSeries(true, true, selfMock);
     expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000, 1000]}, {"config": {"links": []}, "name": "A", "values": [1000, 10]}], "length": 2, "refId": undefined}, {"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000, 1000, 2000, 2000]}, {"config": {"links": []}, "name": "B", "values": [null, null, 2000, 20]}], "length": 4, "refId": undefined}])
   });
 
+
   it('should handle null values correctly', () => {
     selfMock.series = [
       { time: 1000, value: null },
@@ -371,7 +371,7 @@ describe('sql-series. toTimeSeries unit tests', () => {
     ];
     selfMock.keys = [];
 
-    const result = toTimeSeries(false, selfMock);
+    const result = toTimeSeries(false, true, selfMock);
     expect(result).toEqual([{"fields": [{"config": {"links": []}, "name": "time", "type": "time", "values": [1000, 2000]}, {"config": {"links": []}, "name": "value", "values": [null, 20]}], "length": 2, "refId": undefined}]);
   });
 });

diff --git a/src/types/types.ts b/src/types/types.ts
@@ -48,6 +48,7 @@ export interface CHQuery extends DataQuery {
 
   skip_comments?: boolean;
   add_metadata?: boolean;
+  nullifySparse?: boolean;
 
   round?: string;
   intervalFactor?: number;
@@ -88,6 +89,7 @@ export interface CHDataSourceOptions extends DataSourceJsonData {
   adHocHideTableNames?: boolean;
   contextWindowSize?: string;
   useWindowFuncForMacros?: boolean;
+  nullifySparse?: boolean;
 }
 
 /**
@@ -105,4 +107,5 @@ export const DEFAULT_QUERY: CHQuery = {
   showHelp: false,
   showFormattedSQL: false,
   datasourceMode: DatasourceMode.Datasource,
+  nullifySparse: false,
 };
diff --git a/src/views/ConfigEditor/FormParts/DefaultValues/DefaultValues.tsx b/src/views/ConfigEditor/FormParts/DefaultValues/DefaultValues.tsx
@@ -338,6 +338,14 @@ export const DefaultValues = ({ jsonData, newOptions, onSwitchToggle, onFieldCha
               onChange={(e) => onSwitchToggle('useWindowFuncForMacros', e.currentTarget.checked)}
             />
           </InlineField>
+          <InlineField label="Nullify sparse categories" labelWidth={32} style={{ marginLeft: '30px' }}>
+            <InlineSwitch
+              id="nullifySparse"
+              data-testid="nullify-sparse-switch"
+              value={jsonData.nullifySparse ?? false}
+              onChange={(e) => onSwitchToggle('nullifySparse', e.currentTarget.checked)}
+            />
+          </InlineField>
         </>
       )}
     </div>

diff --git a/src/views/QueryEditor/components/QueryHeader/helpers/findDifferences.ts b/src/views/QueryEditor/components/QueryHeader/helpers/findDifferences.ts
@@ -91,6 +91,14 @@ export function findDifferences(query: CHQuery, datasource: CHDataSource) {
         fieldName: 'useWindowFuncForMacros',
       });
     }
+    if (query.nullifySparse !== defaultValues.nullifySparse) {
+      differences.push({
+        key: 'Nullify sparse categories',
+        original: query.nullifySparse?.toString() || 'false',
+        updated: defaultValues.nullifySparse?.toString() || 'false',
+        fieldName: 'nullifySparse',
+      });
+    }
   }
 
   return differences;

diff --git a/src/views/QueryEditor/components/QueryTextEditor/QueryTextEditor.tsx b/src/views/QueryEditor/components/QueryTextEditor/QueryTextEditor.tsx
@@ -13,6 +13,7 @@ import {
   RoundInput,
   MetadataSwitch,
   SkipCommentsSwitch,
+  NullifySparseSwitch,
   UseWindowFunctionSwitch,
   FormatAsSelect,
   ContextWindowSizeSelect,
@@ -80,6 +81,10 @@ export const QueryTextEditor: React.FC<QueryTextEditorProps> = ({
             query={query}
             onChange={() => handlers.handleToggleField('skip_comments')}
           />
+          <NullifySparseSwitch
+            query={query}
+            onChange={() => handlers.handleToggleField('nullifySparse')}
+          />
           <UseWindowFunctionSwitch
             query={query}
             onChange={() => handlers.handleToggleField('useWindowFuncForMacros')}

diff --git a/src/views/QueryEditor/components/QueryTextEditor/components/Switches/NullifySparseSwitch.tsx b/src/views/QueryEditor/components/QueryTextEditor/components/Switches/NullifySparseSwitch.tsx
@@ -0,0 +1,22 @@
+import React from 'react';
+import { InlineField, InlineLabel, InlineSwitch } from '@grafana/ui';
+import { SwitchProps } from '../../types';
+
+export const NullifySparseSwitch: React.FC<SwitchProps> = ({ query, onChange }) => (
+  <InlineField
+    label={
+      <InlineLabel width={18} tooltip="Replace sparse categories with NULL values">
+        Nullify Sparse
+      </InlineLabel>
+    }
+    style={{ height: '100%' }}
+  >
+    <InlineSwitch
+      data-testid="nullify-sparse-switch"
+      width="auto"
+      value={query.nullifySparse}
+      onChange={onChange}
+      transparent
+    />
+  </InlineField>
+); 
diff --git a/src/views/QueryEditor/components/QueryTextEditor/components/index.ts b/src/views/QueryEditor/components/QueryTextEditor/components/index.ts
@@ -6,6 +6,7 @@ export * from './Selects/FormatAsSelect';
 export * from './Selects/ResolutionsInput';
 export * from './Switches/ExtrapolationSwitch';
 export * from './Switches/MetadataSwitch';
+export * from './Switches/NullifySparseSwitch';
 export * from './Switches/SkipCommentsSwitch';
 export * from './Switches/UseWindowFunctionSwitch';
 export * from './Toolbar/ToolbarButtons';
diff --git a/src/views/QueryEditor/components/QueryTextEditor/types.ts b/src/views/QueryEditor/components/QueryTextEditor/types.ts
@@ -8,6 +8,7 @@ export interface Query {
   round?: string;
   add_metadata?: boolean;
   skip_comments?: boolean;
+  nullifySparse?: boolean;
   useWindowFuncForMacros?: boolean;
   format: string;
   contextWindowSize?: string;

diff --git a/src/views/QueryEditor/helpers/initializeQueryDefaults.ts b/src/views/QueryEditor/helpers/initializeQueryDefaults.ts
@@ -13,6 +13,7 @@ export const initializeQueryDefaults = (
     extrapolate: query.extrapolate ?? true,
     skip_comments: query.skip_comments ?? true,
     add_metadata: query.add_metadata ?? true,
+    nullifySparse: query.nullifySparse ?? false,
     useWindowFuncForMacros: query.useWindowFuncForMacros ?? true,
     dateTimeType: query.dateTimeType,
     round: query.round || DEFAULT_ROUND,
@@ -95,6 +96,11 @@ export const initializeQueryDefaults = (
       initializedQuery.contextWindowSize = datasource.defaultValues.contextWindowSize;
     }
 
+    console.log(datasource.defaultValues.nullifySparse, query.nullifySparse, '------')
+    if (datasource.defaultValues.nullifySparse !== undefined && query.nullifySparse === undefined) {
+      initializedQuery.nullifySparse = datasource.defaultValues.nullifySparse;
+    }
+
     onChange({ ...query, ...initializedQuery, initialized: true });
   }
 
@@ -118,6 +124,7 @@ export const initializeQueryDefaultsForVariables = (
     extrapolate: query.extrapolate ?? true,
     skip_comments: query.skip_comments ?? true,
     add_metadata: query.add_metadata ?? true,
+    nullifySparse: query.nullifySparse ?? false,
     useWindowFuncForMacros: query.useWindowFuncForMacros ?? true,
     dateTimeType: query.dateTimeType,
     round: query.round || DEFAULT_ROUND,
@@ -200,6 +207,10 @@ export const initializeQueryDefaultsForVariables = (
       initializedQuery.contextWindowSize = datasource.defaultValues.contextWindowSize;
     }
 
+    if (datasource.defaultValues.nullifySparse !== undefined && query.nullifySparse === undefined) {
+      initializedQuery.nullifySparse = datasource.defaultValues.nullifySparse;
+    }
+
     onChange({ ...query, ...initializedQuery, initialized: true });
   }