feat: denoise search results (#751)

![image](https://github.com/user-attachments/assets/85966fd0-3109-46a3-ab70-56ba06d1ac30)

Author: MikeShi42

packages/app/src/DBSearchPage.tsx

@@ -503,6 +503,18 @@ function DBSearchPage() {
     }
   }, [analysisMode, setIsLive]);
 
+  const [denoiseResults, _setDenoiseResults] = useQueryState(
+    'denoise',
+    parseAsBoolean.withDefault(false),
+  );
+  const setDenoiseResults = useCallback(
+    (value: boolean) => {
+      setIsLive(false);
+      _setDenoiseResults(value);
+    },
+    [setIsLive, _setDenoiseResults],
+  );
+
   const {
     control,
     watch,
@@ -719,11 +731,11 @@ function DBSearchPage() {
   const onTableScroll = useCallback(
     (scrollTop: number) => {
       // If the user scrolls a bit down, kick out of live mode
-      if (scrollTop > 16) {
+      if (scrollTop > 16 && isLive) {
         setIsLive(false);
       }
     },
-    [setIsLive],
+    [isLive, setIsLive],
   );
 
   const onRowExpandClick = useCallback(
@@ -1247,6 +1259,8 @@ function DBSearchPage() {
             >
               <ErrorBoundary message="Unable to render search filters">
                 <DBSearchPageFilters
+                  denoiseResults={denoiseResults}
+                  setDenoiseResults={setDenoiseResults}
                   isLive={isLive}
                   analysisMode={analysisMode}
                   setAnalysisMode={setAnalysisMode}
@@ -1534,25 +1548,31 @@ function DBSearchPage() {
                   </>
                 ) : (
                   <>
-                    {shouldShowLiveModeHint && analysisMode === 'results' && (
-                      <div
-                        className="d-flex justify-content-center"
-                        style={{ height: 0 }}
-                      >
+                    {shouldShowLiveModeHint &&
+                      analysisMode === 'results' &&
+                      denoiseResults != true && (
                         <div
-                          style={{ position: 'relative', top: -20, zIndex: 2 }}
+                          className="d-flex justify-content-center"
+                          style={{ height: 0 }}
                         >
-                          <Button
-                            size="compact-xs"
-                            variant="outline"
-                            onClick={handleResumeLiveTail}
+                          <div
+                            style={{
+                              position: 'relative',
+                              top: -20,
+                              zIndex: 2,
+                            }}
                           >
-                            <i className="bi text-success bi-lightning-charge-fill me-2" />
-                            Resume Live Tail
-                          </Button>
+                            <Button
+                              size="compact-xs"
+                              variant="outline"
+                              onClick={handleResumeLiveTail}
+                            >
+                              <i className="bi text-success bi-lightning-charge-fill me-2" />
+                              Resume Live Tail
+                            </Button>
+                          </div>
                         </div>
-                      </div>
-                    )}
+                      )}
                     {chartConfig &&
                       dbSqlRowTableConfig &&
                       analysisMode === 'results' && (
@@ -1565,6 +1585,7 @@ function DBSearchPage() {
                           queryKeyPrefix={QUERY_KEY_PREFIX}
                           onScroll={onTableScroll}
                           onError={handleTableError}
+                          denoiseResults={denoiseResults}
                         />
                       )}
                   </>

packages/app/src/components/DBRowTable.tsx

@@ -27,7 +27,11 @@ import {
 } from '@hyperdx/common-utils/dist/types';
 import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/utils';
 import { Box, Code, Flex, Text } from '@mantine/core';
-import { FetchNextPageOptions } from '@tanstack/react-query';
+import {
+  FetchNextPageOptions,
+  useQuery,
+  useQueryClient,
+} from '@tanstack/react-query';
 import {
   ColumnDef,
   ColumnResizeMode,
@@ -41,6 +45,7 @@ import { useVirtualizer } from '@tanstack/react-virtual';
 
 import { useTableMetadata } from '@/hooks/useMetadata';
 import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery';
+import { useGroupedPatterns } from '@/hooks/usePatterns';
 import useRowWhere from '@/hooks/useRowWhere';
 import { UNDEFINED_WIDTH } from '@/tableUtils';
 import { FormatTime } from '@/useFormatTime';
@@ -900,6 +905,7 @@ export function DBSqlRowTable({
   isLive = false,
   queryKeyPrefix,
   onScroll,
+  denoiseResults = false,
 }: {
   config: ChartConfigWithDateRange;
   onRowExpandClick?: (where: string) => void;
@@ -909,6 +915,7 @@ export function DBSqlRowTable({
   isLive?: boolean;
   onScroll?: (scrollTop: number) => void;
   onError?: (error: Error | ClickHouseQueryError) => void;
+  denoiseResults?: boolean;
 }) {
   const mergedConfig = useConfigWithPrimaryAndPartitionKey(config);
 
@@ -964,7 +971,7 @@ export function DBSqlRowTable({
       });
       return newRow;
     });
-  }, [data, objectTypeColumns]);
+  }, [data, objectTypeColumns, columnMap]);
 
   const aliasMap = chSqlToAliasMap(data?.chSql ?? { sql: '', params: {} });
 
@@ -983,23 +990,117 @@ export function DBSqlRowTable({
     }
   }, [isError, onError, error]);
 
+  const patternColumn = columns[columns.length - 1];
+  const groupedPatterns = useGroupedPatterns({
+    config,
+    samples: 10_000,
+    bodyValueExpression: patternColumn ?? '',
+    totalCount: undefined,
+    enabled: denoiseResults,
+  });
+  const noisyPatterns = useQuery({
+    queryKey: ['noisy-patterns', config],
+    queryFn: async () => {
+      return Object.values(groupedPatterns.data).filter(
+        p => p.count / (groupedPatterns.sampledRowCount ?? 1) > 0.1,
+      );
+    },
+    enabled:
+      denoiseResults &&
+      groupedPatterns.data != null &&
+      Object.values(groupedPatterns.data).length > 0 &&
+      groupedPatterns.miner != null,
+  });
+  const noisyPatternIds = useMemo(() => {
+    return noisyPatterns.data?.map(p => p.id) ?? [];
+  }, [noisyPatterns.data]);
+
+  const queryClient = useQueryClient();
+
+  const denoisedRows = useQuery({
+    queryKey: [
+      'denoised-rows',
+      config,
+      processedRows,
+      noisyPatternIds,
+      patternColumn,
+    ],
+    queryFn: async () => {
+      // No noisy patterns, so no need to denoise
+      if (noisyPatternIds.length === 0) {
+        return processedRows;
+      }
+
+      const matchedLogs = await groupedPatterns.miner?.matchLogs(
+        processedRows.map(row => row[patternColumn]),
+      );
+      return processedRows.filter((row, i) => {
+        const match = matchedLogs?.[i];
+        return !noisyPatternIds.includes(`${match}`);
+      });
+    },
+    placeholderData: (previousData, previousQuery) => {
+      // If it's the same search, but new data, return the previous data while we load
+      if (
+        previousQuery?.queryKey?.[0] === 'denoised-rows' &&
+        previousQuery?.queryKey?.[1] === config
+      ) {
+        return previousData;
+      }
+      return undefined;
+    },
+    enabled:
+      denoiseResults &&
+      noisyPatterns.isSuccess &&
+      processedRows.length > 0 &&
+      groupedPatterns.miner != null,
+  });
+
+  const isLoading = denoiseResults
+    ? isFetching ||
+      denoisedRows.isFetching ||
+      noisyPatterns.isFetching ||
+      groupedPatterns.isLoading
+    : isFetching;
+
   return (
-    <RawLogTable
-      isLive={isLive}
-      wrapLines={false}
-      displayedColumns={columns}
-      highlightedLineId={highlightedLineId}
-      rows={processedRows}
-      isLoading={isFetching}
-      fetchNextPage={fetchNextPage}
-      // onPropertySearchClick={onPropertySearchClick}
-      hasNextPage={hasNextPage}
-      onRowExpandClick={_onRowExpandClick}
-      onScroll={onScroll}
-      generateRowId={getRowWhere}
-      isError={isError}
-      error={error ?? undefined}
-      columnTypeMap={columnMap}
-    />
+    <>
+      {denoiseResults && (
+        <Box mb="xxs" px="sm" mt="-24px">
+          <Text fw="bold" fz="xs" mb="xxs">
+            Removed Noisy Event Patterns
+          </Text>
+          <Box mah={100} style={{ overflow: 'auto' }}>
+            {noisyPatterns.data?.map(p => (
+              <Text c="gray.3" fz="xs" key={p.id}>
+                {p.pattern}
+              </Text>
+            ))}
+            {noisyPatternIds.length === 0 && (
+              <Text c="gray.3" fz="xs">
+                No noisy patterns found
+              </Text>
+            )}
+          </Box>
+        </Box>
+      )}
+      <RawLogTable
+        isLive={isLive}
+        wrapLines={false}
+        displayedColumns={columns}
+        highlightedLineId={highlightedLineId}
+        rows={denoiseResults ? (denoisedRows?.data ?? []) : processedRows}
+        isLoading={isLoading}
+        fetchNextPage={fetchNextPage}
+        // onPropertySearchClick={onPropertySearchClick}
+        hasNextPage={hasNextPage}
+        onRowExpandClick={_onRowExpandClick}
+        onScroll={onScroll}
+        generateRowId={getRowWhere}
+        isError={isError}
+        error={error ?? undefined}
+        columnTypeMap={columnMap}
+      />
+    </>
   );
 }

packages/app/src/components/DBSearchPageFilters.tsx

@@ -326,13 +326,17 @@ export const DBSearchPageFilters = ({
   setAnalysisMode,
   sourceId,
   showDelta,
+  denoiseResults,
+  setDenoiseResults,
 }: {
   analysisMode: 'results' | 'delta' | 'pattern';
   setAnalysisMode: (mode: 'results' | 'delta' | 'pattern') => void;
   isLive: boolean;
   chartConfig: ChartConfigWithDateRange;
   sourceId?: string;
   showDelta: boolean;
+  denoiseResults: boolean;
+  setDenoiseResults: (denoiseResults: boolean) => void;
 } & FilterStateHook) => {
   const { toggleFilterPin, isFilterPinned } = usePinnedFilters(
     sourceId ?? null,
@@ -489,6 +493,28 @@ export const DBSearchPageFilters = ({
             )}
           </Flex>
 
+          {analysisMode === 'results' && (
+            <Checkbox
+              size={13 as any}
+              checked={denoiseResults}
+              ms="6px"
+              label={
+                <Tooltip
+                  openDelay={200}
+                  color="gray"
+                  position="right"
+                  withArrow
+                  label="Denoise results will visually remove events matching common event patterns from the results table."
+                >
+                  <Text size="xs" c="gray.3" mt="-1px">
+                    <i className="bi bi-noise-reduction"></i> Denoise Results
+                  </Text>
+                </Tooltip>
+              }
+              onChange={() => setDenoiseResults(!denoiseResults)}
+            />
+          )}
+
           {isLoading || isFacetsLoading ? (
             <Flex align="center" justify="center">
               <Loader size="xs" color="gray" />