feat(logs): global search

packages/logs/lib/models/messages.ts

@@ -87,9 +87,11 @@ export async function listOperations(opts: {
             should: []
         }
     };
+
     if (opts.environmentId) {
         (query.bool!.must as estypes.QueryDslQueryContainer[]).push({ term: { environmentId: opts.environmentId } });
     }
+
     if (opts.states && (opts.states.length > 1 || opts.states[0] !== 'all')) {
         // Where or
         (query.bool!.must as estypes.QueryDslQueryContainer[]).push({
@@ -100,6 +102,7 @@ export async function listOperations(opts: {
             }
         });
     }
+
     if (opts.integrations && (opts.integrations.length > 1 || opts.integrations[0] !== 'all')) {
         // Where or
         (query.bool!.must as estypes.QueryDslQueryContainer[]).push({
@@ -110,6 +113,7 @@ export async function listOperations(opts: {
             }
         });
     }
+
     if (opts.connections && (opts.connections.length > 1 || opts.connections[0] !== 'all')) {
         // Where or
         (query.bool!.must as estypes.QueryDslQueryContainer[]).push({
@@ -120,6 +124,7 @@ export async function listOperations(opts: {
             }
         });
     }
+
     if (opts.syncs && (opts.syncs.length > 1 || opts.syncs[0] !== 'all')) {
         // Where or
         (query.bool!.must as estypes.QueryDslQueryContainer[]).push({
@@ -130,6 +135,7 @@ export async function listOperations(opts: {
             }
         });
     }
+
     if (opts.types && (opts.types.length > 1 || opts.types[0] !== 'all')) {
         const types: estypes.QueryDslQueryContainer[] = [];
         for (const couple of opts.types) {
@@ -147,6 +153,7 @@ export async function listOperations(opts: {
             }
         });
     }
+
     if (opts.period) {
         (query.bool!.must as estypes.QueryDslQueryContainer[]).push({
             range: {
@@ -271,10 +278,7 @@ export async function listMessages(opts: {
     cursorAfter?: string | null | undefined;
 }): Promise<ListMessages> {
     const query: estypes.QueryDslQueryContainer = {
-        bool: {
-            must: [{ term: { parentId: opts.parentId } }],
-            should: []
-        }
+        bool: { must: [{ term: { parentId: opts.parentId } }], should: [] }
     };
 
     if (opts.states && (opts.states.length > 1 || opts.states[0] !== 'all')) {
@@ -341,6 +345,46 @@ export async function listMessages(opts: {
     };
 }
 
+/**
+ * This method is searching logs inside each operations, returning a list of matching operations.
+ */
+export async function searchForMessagesInsideOperations(opts: { search: string; operationsIds: string[] }): Promise<{
+    items: { key: string; doc_count: number }[];
+}> {
+    const query: estypes.QueryDslQueryContainer = {
+        bool: {
+            must: [
+                { exists: { field: 'parentId' } },
+                {
+                    match_phrase_prefix: { meta_search: { query: opts.search } }
+                },
+                { terms: { parentId: opts.operationsIds } }
+            ]
+        }
+    };
+
+    const res = await client.search<
+        OperationRow,
+        {
+            parentIdAgg: estypes.AggregationsTermsAggregateBase<{ key: string; doc_count: number }>;
+        }
+    >({
+        index: indexMessages.index,
+        size: 0,
+        sort: [{ createdAt: 'desc' }, 'id'],
+        track_total_hits: false,
+        query,
+        aggs: {
+            // We aggregate because we can have N match per operation
+            parentIdAgg: { terms: { size: opts.operationsIds.length + 1, field: 'parentId' } }
+        }
+    });
+
+    const aggs = res.aggregations!['parentIdAgg']['buckets'];
+
+    return { items: aggs as any };
+}
+
 /**
  * List filters
  */

packages/server/lib/controllers/v1/logs/searchOperations.ts

@@ -9,6 +9,7 @@ import type { SearchOperations } from '@nangohq/types';
 
 const validation = z
     .object({
+        search: z.string().max(256).optional(),
         limit: z.number().max(500).optional().default(100),
         states: z
             .array(z.enum(['all', 'waiting', 'running', 'success', 'failed', 'timeout', 'cancelled']))
@@ -76,6 +77,7 @@ export const searchOperations = asyncWrapper<SearchOperations>(async (req, res)
 
     const env = res.locals['environment'];
     const body: SearchOperations['Body'] = val.data;
+
     const rawOps = await model.listOperations({
         accountId: env.account_id,
         environmentId: env.id,
@@ -88,6 +90,11 @@ export const searchOperations = asyncWrapper<SearchOperations>(async (req, res)
         period: body.period,
         cursor: body.cursor
     });
+    if (body.search && rawOps.items.length > 0) {
+        const bucket = await model.searchForMessagesInsideOperations({ search: body.search, operationsIds: rawOps.items.map((op) => op.id) });
+        const matched = new Set(bucket.items.map((item) => item.key));
+        rawOps.items = rawOps.items.filter((item) => matched.has(item.id));
+    }
 
     res.status(200).send({
         data: rawOps.items,

packages/types/lib/logs/api.ts

@@ -12,6 +12,7 @@ export type SearchOperations = Endpoint<{
     Path: '/api/v1/logs/operations';
     Querystring: { env: string };
     Body: {
+        search?: string | undefined;
         limit?: number;
         states?: SearchOperationsState[];
         types?: SearchOperationsType[];

packages/webapp/src/components/ui/button/Button.tsx

@@ -66,8 +66,8 @@ export const Button = forwardRef<HTMLButtonElement, ButtonProps>(function Button
         <button ref={ref} className={cn(buttonStyles({ variant, size }), 'relative flex gap-2 items-center', className, isLoading && 'opacity-0')} {...props}>
             {children}
             {isLoading && (
-                <div className="py-1.5 h-full">
-                    <Loader className="animate-spin h-full" />
+                <div className={cn('h-full w-4 flex items-center justify-center', size === 'xs' && 'w-3')}>
+                    <Loader className="animate-spin h-full w-full" />
                 </div>
             )}
         </button>

packages/webapp/src/pages/Logs/components/SearchAllOperations.tsx

@@ -1,9 +1,10 @@
+import { IconSearch, IconX } from '@tabler/icons-react';
 import { keepPreviousData, useInfiniteQuery } from '@tanstack/react-query';
 import { flexRender, getCoreRowModel, useReactTable } from '@tanstack/react-table';
 import { useVirtualizer } from '@tanstack/react-virtual';
 import { parseAsArrayOf, parseAsBoolean, parseAsString, parseAsStringEnum, parseAsStringLiteral, parseAsTimestamp, useQueryState } from 'nuqs';
 import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react';
-import { useInterval, useMount, useWindowSize } from 'react-use';
+import { useDebounce, useInterval, useMount, useWindowSize } from 'react-use';
 
 import { DatePicker } from './DatePicker';
 import { SearchableMultiSelect } from './SearchableMultiSelect';
@@ -12,6 +13,8 @@ import { MultiSelect } from '../../../components/MultiSelect';
 import { Skeleton } from '../../../components/ui/Skeleton';
 import Spinner from '../../../components/ui/Spinner';
 import * as Table from '../../../components/ui/Table';
+import { Button } from '../../../components/ui/button/Button';
+import { Input } from '../../../components/ui/input/Input';
 import { queryClient, useStore } from '../../../store';
 import { columns, defaultLimit, refreshInterval, statusOptions, typesList } from '../constants';
 import { OperationRow } from './OperationRow';
@@ -28,6 +31,7 @@ interface Props {
     onSelectOperation: (open: boolean, operationId: string) => void;
 }
 
+const parseSearch = parseAsString.withDefault('');
 const parseLive = parseAsBoolean.withDefault(true).withOptions({ history: 'push' });
 const parseStates = parseAsArrayOf(parseAsStringLiteral(statusOptions.map((opt) => opt.value)), ',')
     .withDefault(['all'])
@@ -48,6 +52,7 @@ export const SearchAllOperations: React.FC<Props> = ({ onSelectOperation }) => {
     const windowSize = useWindowSize();
 
     // --- Data fetch
+    const [search, setSearch] = useQueryState('search', parseSearch);
     const [isLive, setLive] = useQueryState('live', parseLive);
     const [states, setStates] = useQueryState('states', parseStates);
     const [types, setTypes] = useQueryState('types', parseTypes);
@@ -58,6 +63,17 @@ export const SearchAllOperations: React.FC<Props> = ({ onSelectOperation }) => {
 
     // We optimize the refresh and memory when the users is waiting for new operations (= scroll is on top)
     const [isScrollTop, setIsScrollTop] = useState(false);
+    const [manualLoadMore, setManualLoadMore] = useState(true);
+
+    const [debouncedSearch, setDebouncedSearch] = useState<string | undefined>(() => search);
+    useDebounce(
+        () => {
+            setDebouncedSearch(search);
+            setManualLoadMore(search !== ''); // Because it can spam the backend we disable infinite scroll
+        },
+        250,
+        [search]
+    );
 
     /**
      * Because we haven't build a forward pagination it's currently impossible to have a proper infinite scroll both way and a live refresh
@@ -69,14 +85,14 @@ export const SearchAllOperations: React.FC<Props> = ({ onSelectOperation }) => {
      * - When you scroll to the bottom, we disable interval refresh, and the infiniteQuery load the next pages
      * - When you scroll back to the top we trim all the pages so they don't get refresh, and we re-enable the interval refresh
      */
-    const { data, isLoading, isFetching, fetchNextPage, refetch } = useInfiniteQuery<
+    const { data, isLoading, isFetching, isFetchingNextPage, hasNextPage, fetchNextPage, refetch } = useInfiniteQuery<
         SearchOperations['Success'],
         SearchOperations['Errors'],
         { pages: SearchOperations['Success'][] },
         unknown[],
         string | null
     >({
-        queryKey: [env, 'logs:operations:infinite', states, types, integrations, connections, syncs, period],
+        queryKey: [env, 'logs:operations:infinite', states, types, integrations, connections, syncs, period, debouncedSearch],
         queryFn: async ({ pageParam, signal }) => {
             let periodCopy: SearchOperations['Body']['period'];
             // Slide the window automatically when live
@@ -97,8 +113,12 @@ export const SearchAllOperations: React.FC<Props> = ({ onSelectOperation }) => {
                     connections,
                     syncs,
                     period: periodCopy,
-                    limit: defaultLimit,
-                    cursor: pageParam
+                    // Search is post-filtering the list of operations, it can change the actual number of returned operations
+                    // It's more efficient to increase the limit of pre-filtered operations we get, and do less round trip
+                    // The "drawbacks" is that if every operations are matching, it can be slower and return more rows that expected
+                    limit: debouncedSearch ? defaultLimit * 10 : defaultLimit,
+                    cursor: pageParam,
+                    search: debouncedSearch
                 } satisfies SearchOperations['Body']),
                 signal
             });
@@ -124,7 +144,7 @@ export const SearchAllOperations: React.FC<Props> = ({ onSelectOperation }) => {
         async () => {
             await refetch({ cancelRefetch: true });
         },
-        isLive && isScrollTop ? refreshInterval : null
+        isLive && isScrollTop && !debouncedSearch ? refreshInterval : null
     );
     useMount(async () => {
         // We can't use standard refetchOnMount because it will refresh every pages so we force refresh the first one
@@ -134,17 +154,20 @@ export const SearchAllOperations: React.FC<Props> = ({ onSelectOperation }) => {
     });
 
     const trim = useCallback(() => {
-        queryClient.setQueryData([env, 'logs:operations:infinite', states, types, integrations, connections, syncs, period], (oldData: any) => {
-            if (!oldData || !oldData.pages || oldData.pages.length <= 1) {
-                return oldData;
-            }
+        queryClient.setQueryData(
+            [env, 'logs:operations:infinite', states, types, integrations, connections, syncs, period, debouncedSearch],
+            (oldData: any) => {
+                if (!oldData || !oldData.pages || oldData.pages.length <= 1) {
+                    return oldData;
+                }
 
-            return {
-                ...oldData,
-                pages: [oldData.pages[0]],
-                pageParams: [oldData.pageParams[0]]
-            };
-        });
+                return {
+                    ...oldData,
+                    pages: [oldData.pages[0]],
+                    pageParams: [oldData.pageParams[0]]
+                };
+            }
+        );
     }, [env, states, types, integrations, connections, syncs, period]);
 
     const flatData = useMemo<OperationRowType[]>(() => {
@@ -183,6 +206,12 @@ export const SearchAllOperations: React.FC<Props> = ({ onSelectOperation }) => {
             }
 
             const { scrollHeight, scrollTop, clientHeight } = containerRefElement;
+
+            // We don't want to refresh or trim the pages when searching
+            if (manualLoadMore) {
+                return;
+            }
+
             // once the user has scrolled within 200px of the bottom of the table, fetch more data if we can
             if (scrollHeight - scrollTop - clientHeight < 200 && !isFetching && totalFetched < totalOperations) {
                 void fetchNextPage({ cancelRefetch: true });
@@ -195,11 +224,29 @@ export const SearchAllOperations: React.FC<Props> = ({ onSelectOperation }) => {
                 setIsScrollTop(false);
             }
         },
-        [fetchNextPage, isFetching, totalFetched, totalOperations, isLive, isScrollTop]
+        [fetchNextPage, isFetching, totalFetched, totalOperations, isLive, isScrollTop, manualLoadMore]
     );
     useEffect(() => {
         fetchMoreOnBottomReached(tableContainerRef.current);
     }, [fetchMoreOnBottomReached]);
+    const onClickLoadMore = useCallback(() => {
+        setManualLoadMore(false);
+    }, []);
+    useEffect(() => {
+        // When searching, pagination is incomplete it can be daunting to manually click the button a lot
+        // So we disable manual load more until we find a next page
+        if (!debouncedSearch || manualLoadMore) {
+            return;
+        }
+
+        if (!hasNextPage) {
+            setManualLoadMore(true);
+        }
+
+        if (data?.pages && data.pages.length > 0 && data.pages.at(-1)!.data.length > 0) {
+            setManualLoadMore(true);
+        }
+    }, [data, hasNextPage]);
 
     // --- Period
     const onPeriodChange = (range: DateRange, live: boolean) => {
@@ -216,15 +263,31 @@ export const SearchAllOperations: React.FC<Props> = ({ onSelectOperation }) => {
                 </div>
             </div>
             <div className="flex gap-2 justify-between mb-4">
-                <div className="w-full"> </div>
+                <div className="w-full">
+                    <Input
+                        before={<IconSearch stroke={1} size={16} />}
+                        after={
+                            search && (
+                                <Button variant={'icon'} size={'xs'} onClick={() => setSearch('')}>
+                                    <IconX stroke={1} size={18} />
+                                </Button>
+                            )
+                        }
+                        placeholder="Search logs..."
+                        className="border-grayscale-900"
+                        onChange={(e) => setSearch(e.target.value)}
+                        inputSize={'sm'}
+                        value={search}
+                    />
+                </div>
                 <div className="flex gap-2">
                     <MultiSelect label="Status" options={statusOptions} selected={states} defaultSelect={['all']} onChange={setStates} all />
                     <TypesSelect selected={types} onChange={setTypes} />
                     <SearchableMultiSelect label="Integration" selected={integrations} category={'integration'} onChange={setIntegrations} max={20} />
                     <SearchableMultiSelect label="Connection" selected={connections} category={'connection'} onChange={setConnections} max={20} />
                     <SearchableMultiSelect label="Script" selected={syncs} category={'syncConfig'} onChange={setSyncs} max={20} />
 
-                    <DatePicker isLive={isLive} period={{ from: period[0], to: period[1] }} onChange={onPeriodChange} />
+                    <DatePicker isLive={!manualLoadMore && isLive} period={{ from: period[0], to: period[1] }} onChange={onPeriodChange} />
                 </div>
             </div>
             <div
@@ -257,6 +320,13 @@ export const SearchAllOperations: React.FC<Props> = ({ onSelectOperation }) => {
 
                     {flatData.length > 0 && <TableBody table={table} tableContainerRef={tableContainerRef} onSelectOperation={onSelectOperation} />}
 
+                    {flatData.length > 0 && hasNextPage && (
+                        <Button onClick={onClickLoadMore} variant={'emptyFaded'} className="justify-center mt-4 text-s" isLoading={isFetchingNextPage}>
+                            Load more...
+                        </Button>
+                    )}
+                    {flatData.length > 0 && !hasNextPage && <div className="text-xs text-grayscale-500 p-4 mt-2">Nothing more to load...</div>}
+
                     {isLoading && (
                         <Table.Body>
                             <Table.Row>
@@ -271,7 +341,7 @@ export const SearchAllOperations: React.FC<Props> = ({ onSelectOperation }) => {
                         </Table.Body>
                     )}
 
-                    {!isFetching && flatData.length <= 0 && (
+                    {!isLoading && flatData.length <= 0 && (
                         <Table.Body>
                             <Table.Row className="hover:bg-transparent flex absolute w-full">
                                 <Table.Cell colSpan={columns.length} className="h-24 text-center p-0 pt-4 w-full">