Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions src/components/common/infinite-scroll.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { PropsWithChildren } from "react"
import type { PropsWithChildren, ReactNode } from "react"
import { useEffect, useRef } from "react"
import { useTab } from "@/tab-provider"
import { IconLoader2 } from "@tabler/icons-react"
Expand All @@ -10,9 +10,16 @@ import { ScrollArea } from "@/components/ui/scroll-area"
export const InfiniteScroll = ({
query,
children,
// When false, scrolling / viewport-fill won't auto-fetch the next page. Used to
// pause runaway fetching (e.g. a type filter that matches nothing) and hand
// control to an explicit continue action rendered via `endSlot`.
autoFetch = true,
endSlot,
...props
}: PropsWithChildren<{
query: UseInfiniteQueryResult
autoFetch?: boolean
endSlot?: ReactNode
}> &
React.ComponentProps<typeof ScrollArea>) => {
const { active } = useTab()
Expand All @@ -21,6 +28,7 @@ export const InfiniteScroll = ({

// Fetch more on scroll
const handleScroll = (e: React.UIEvent<HTMLDivElement>) => {
if (!autoFetch) return
const { scrollTop, clientHeight, scrollHeight } = e.currentTarget
if (scrollTop + clientHeight > scrollHeight - 100) {
if (query.isFetching || !query.hasNextPage) {
Expand All @@ -32,6 +40,7 @@ export const InfiniteScroll = ({

// Check if viewport is filled and fetch more if needed
const checkAndFetchMore = () => {
if (!autoFetch) return
if (!scrollRef.current || !contentRef.current) return

const viewportHeight = scrollRef.current.clientHeight
Expand Down Expand Up @@ -64,7 +73,7 @@ export const InfiniteScroll = ({
{children}

<div className="flex h-[100px] justify-center py-2 text-zinc-300">
{query.isFetching && <IconLoader2 className="animate-spin" size={16} />}
{query.isFetching ? <IconLoader2 className="animate-spin" size={16} /> : endSlot}
</div>
</div>
</ScrollArea>
Expand Down
10 changes: 9 additions & 1 deletion src/components/databrowser/components/sidebar/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@ import { InfiniteScroll } from "../../../common/infinite-scroll"
import { useKeys } from "../../hooks/use-keys"
import { Empty } from "./empty"
import { KeysList } from "./keys-list"
import { KeepScanningSlot, ScanEmpty } from "./scan-status"
import { LoadingSkeleton } from "./skeleton-buttons"

export function Sidebar() {
const { keys, query } = useKeys()
const { keys, query, scan } = useKeys()

return (
<div className="relative flex h-full flex-col gap-2">
Expand All @@ -15,12 +16,19 @@ export function Sidebar() {
// Infinite scroll already has a loader at the bottom
<InfiniteScroll
query={query}
// Pause runaway auto-fetching when a sparse type filter stops matching;
// the user resumes via the "keep scanning" slot.
autoFetch={!scan.paused}
endSlot={scan.paused ? <KeepScanningSlot /> : undefined}
disableRoundedInherit
className="h-full min-h-0 rounded-xl bg-zinc-100 px-2 py-5 pr-4 dark:bg-zinc-200"
scrollBarClassName="py-5"
>
<KeysList />
</InfiniteScroll>
) : query.hasNextPage ? (
// No matches yet, but keyspace remains (e.g. sparse type filter).
<ScanEmpty />
) : (
<Empty />
)}
Expand Down
59 changes: 59 additions & 0 deletions src/components/databrowser/components/sidebar/scan-status.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import { useTab } from "@/tab-provider"
import { IconLoader2 } from "@tabler/icons-react"

import { useKeys } from "../../hooks/use-keys"

const formatScanned = (n: number) => {
if (n >= 1_000_000) return `~${(n / 1_000_000).toFixed(1)}M`
if (n >= 1000) return `~${(n / 1000).toFixed(1)}K`
return `${n}`
}

const ScanMessage = () => {
const { scan } = useKeys()
const { search } = useTab()
const typeLabel = search.type ? `${search.type} ` : ""

return (
<p className="text-balance text-sm text-zinc-500">
No {typeLabel}keys found in the first {formatScanned(scan.scannedKeys)} keys scanned.
</p>
)
}

const KeepScanningButton = () => {
const { query } = useKeys()

return (
<button
onClick={() => query.fetchNextPage()}
disabled={query.isFetchingNextPage}
className="inline-flex items-center gap-1.5 rounded-lg border border-zinc-300 bg-white px-3 py-1.5 text-sm font-medium text-zinc-700 transition-colors hover:bg-zinc-50 disabled:opacity-60"
>
{query.isFetchingNextPage && <IconLoader2 className="size-4 animate-spin" />}
{query.isFetchingNextPage ? "Scanning…" : "Keep scanning"}
</button>
)
}

/**
* Sidebar body state when a type filter has matched nothing yet but there is
* still keyspace left to scan. Replaces the infinite SCAN loop with an explicit,
* user-driven continue.
*/
export const ScanEmpty = () => (
<div className="flex h-full w-full items-center justify-center rounded-md border bg-white px-4 py-6 text-center">
<div className="flex flex-col items-center gap-4">
<ScanMessage />
<KeepScanningButton />
</div>
</div>
)

/** Bottom-of-list slot when some matches are shown but auto-scan has paused. */
export const KeepScanningSlot = () => (
<div className="flex flex-col items-center gap-2">
<ScanMessage />
<KeepScanningButton />
</div>
)
69 changes: 53 additions & 16 deletions src/components/databrowser/hooks/use-keys.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,24 @@ const KeysContext = createContext<
| {
keys: RedisKey[]
query: UseInfiniteQueryResult
scan: { paused: boolean; scannedKeys: number }
}
| undefined
>(undefined)

export const FETCH_KEYS_QUERY_KEY = "use-fetch-keys"

const SCAN_COUNTS = [100, 300, 500]
// Upstash REST caps SCAN at ~1000 examined/returned keys per call regardless of
// COUNT (measured), so escalating past 1000 is pointless. We still ramp up from a
// small count for a snappy first response, then sit at the 1000 ceiling.
const SCAN_COUNTS = [100, 500, 1000]

// Hard cap on SCAN round-trips per page fetch. On a large keyspace a type filter
// can match nothing for millions of keys; without this the query loops effectively
// forever. When the budget is spent without finding a key we yield an empty page
// and let the user explicitly continue (see `scan.paused`).
const MAX_SCANS_PER_FETCH = 20

type ScanResult = { cursor: string; keys: { key: string; type: DataType; score?: number }[] }

export const KeysProvider = ({ children }: PropsWithChildren) => {
Expand Down Expand Up @@ -153,22 +164,36 @@ export const KeysProvider = ({ children }: PropsWithChildren) => {
}

/**
* Keeps scanning until a result shows up.
* Keeps scanning until a key shows up, the keyspace is exhausted, or this
* fetch's request budget (MAX_SCANS_PER_FETCH) is spent.
*
* When the db is large and sparse for the active type filter, matching keys can
* be millions of keys apart. Rather than loop until we find one (which can mean
* tens of thousands of requests), we stop after a bounded number of round-trips
* and report back so the UI can pause and let the user continue on demand.
*
* When a db is sparse and the type argument is used, it could take a lot of
* requests to find those sparse keys. Best method is to increase the count
* argument when no result is being returned to decrease the number of
* requests.
* `signal` is React Query's abort signal; we bail between round-trips so a
* filter/type change cancels the in-flight scan instead of churning in the
* background.
*/
const scanUntilAvailable = async (cursor: string) => {
let i = 0
while (true) {
const [newCursor, values] = await performScan(SCAN_COUNTS[i] ?? SCAN_COUNTS.at(-1), cursor)
cursor = newCursor
i++
const scanUntilAvailable = async (cursor: string, signal?: AbortSignal) => {
let scannedKeys = 0
for (let i = 0; ; i++) {
if (signal?.aborted) throw new DOMException("Scan aborted", "AbortError")

if (values.length > 0 || cursor === "0") {
return [cursor, values] as const
const count = SCAN_COUNTS[i] ?? SCAN_COUNTS.at(-1)
const [newCursor, values] = await performScan(count, cursor)
cursor = newCursor
// SCAN examines at most ~1000 keys per call on Upstash REST (see SCAN_COUNTS).
scannedKeys += Math.min(count, 1000)

const exhausted = cursor === "0"
const budgetSpent = i + 1 >= MAX_SCANS_PER_FETCH
if (values.length > 0 || exhausted || budgetSpent) {
// paused === we spent the budget without finding anything and there is
// still more keyspace left to scan.
const paused = budgetSpent && values.length === 0 && !exhausted
return { cursor, values, scannedKeys, paused }
}
}
}
Expand All @@ -178,8 +203,8 @@ export const KeysProvider = ({ children }: PropsWithChildren) => {
enabled: isQueryEnabled,

initialPageParam: "0",
queryFn: async ({ pageParam: lastCursor }) => {
const [cursor, values] = await scanUntilAvailable(lastCursor)
queryFn: async ({ pageParam: lastCursor, signal }) => {
const { cursor, values, scannedKeys, paused } = await scanUntilAvailable(lastCursor, signal)

const keys: RedisKey[] = values.map((value) => [value.key, value.type, value.score])

Expand All @@ -192,6 +217,8 @@ export const KeysProvider = ({ children }: PropsWithChildren) => {
cursor: cursor === "0" ? undefined : cursor,
keys,
hasNextPage: cursor !== "0",
scannedKeys,
paused,
}
},
meta: {
Expand All @@ -218,6 +245,15 @@ export const KeysProvider = ({ children }: PropsWithChildren) => {
return dedupedKeys
}, [query.data])

const scan = useMemo(() => {
const pages = query.data?.pages ?? []
return {
// Latest fetch spent its budget without a match — wait for the user to continue.
paused: Boolean(pages.at(-1)?.paused),
scannedKeys: pages.reduce((sum, page) => sum + (page.scannedKeys ?? 0), 0),
}
}, [query.data])

return (
<KeysContext.Provider
value={{
Expand All @@ -226,6 +262,7 @@ export const KeysProvider = ({ children }: PropsWithChildren) => {
...query,
isLoading: query.isLoading || isIndexDetailsLoading,
} as UseInfiniteQueryResult,
scan,
}}
>
{children}
Expand Down
Loading