fix(historical): EOD table lookup, Llama miss, persist today

src/routes/historical/exact.ts

@@ -27,6 +27,7 @@ export async function handleHistorical(
   const source = parseOptionalSource(new URL(request.url).searchParams.get('source'))
 
   const record = await getExactHistoricalPrice(pool, { chain, token, timestamp }, source)
+
   if (!record && !source) {
     const chainId = chainNameToId(chain)
     if (chainId !== undefined) {

src/routes/historical/shared.ts

@@ -3,7 +3,13 @@ import { DEFI_LLAMA_SEARCH_WIDTH_SECONDS } from '../../clients/defillama'
 import { insertTokenPrices } from '../../db'
 import { ensure } from '../../http'
 import type { BatchHistoricalResponseCoin, ExactPriceRecord, HistoricalRequestTuple, RangeRequest } from '../../types'
-import { isClosedDay, normalizedDaysInRange, parseTokenKey } from '../../utils'
+import {
+  currentUtcDayEnd,
+  normalizedDaysInRange,
+  normalizeToEndOfDay,
+  parseTokenKey,
+  toFetchTimestamp
+} from '../../utils'
 
 export interface ResolvedPriceRecord extends ExactPriceRecord {
   /** Timestamp of the underlying observation, not of the day it is keyed under. */
@@ -27,20 +33,23 @@ function isWithinObservationWindow(record: ResolvedPriceRecord): boolean {
   if (record.source === 'chainlink') {
     return true
   }
-  return Math.abs(record.observedAt - record.timestamp) <= DEFI_LLAMA_SEARCH_WIDTH_SECONDS
+  return Math.abs(record.observedAt - toFetchTimestamp(record.timestamp)) <= DEFI_LLAMA_SEARCH_WIDTH_SECONDS
+}
+
+function isPersistableDay(timestamp: number): boolean {
+  return normalizeToEndOfDay(timestamp) <= currentUtcDayEnd()
 }
 
 /**
- * Best-effort request-path persistence. Only closed past days are written:
- * a current- or future-day key was resolved at "now", and writing it under the
- * day-end key would freeze that intraday value as the day's permanent close
- * once the row turns immutable at midnight (the invariant src/routes/spot.ts
- * documents). A write failure is logged and swallowed — the response already
+ * Best-effort request-path persistence. Closed past days and the current UTC
+ * day are written (today as a mutable row) so a DeFiLlama fill is a table hit
+ * for the rest of the day and does not 429-stampede. Future-day keys are not
+ * written. A write failure is logged and swallowed — the response already
  * holds the prices, and a persistence fault must not turn a serveable 200 into
  * a 500.
  */
 export async function persistResolvedPrices(pool: Pool, records: ResolvedPriceRecord[]): Promise<number> {
-  const rows = records.filter((record) => isClosedDay(record.timestamp) && isWithinObservationWindow(record))
+  const rows = records.filter((record) => isPersistableDay(record.timestamp) && isWithinObservationWindow(record))
   if (rows.length === 0) {
     return 0
   }

test/open-day-last-close.test.ts

@@ -0,0 +1,193 @@
+import type { Pool } from '@neondatabase/serverless'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { CACHE_CONTROL_IMMUTABLE, CACHE_CONTROL_TODAY } from '../src/cache'
+import type { HistoricalSourceRegistry } from '../src/registries'
+import { handleBatchHistorical } from '../src/routes/historical/batch'
+import { handleHistorical } from '../src/routes/historical/exact'
+import type { Env } from '../src/types'
+import { normalizeToEndOfDay } from '../src/utils'
+
+const RAW = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'
+const CHECKSUM = '0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2'
+const TOKEN_KEY = `ethereum:${RAW}`
+const CHECKSUM_KEY = `Ethereum:${CHECKSUM}`
+const ENV: Env = { DATABASE_URL: 'postgres://x' }
+const PAST = 1695254399
+const NOW = 1_787_911_200
+const TODAY = normalizeToEndOfDay(NOW)
+
+function exactRequest() {
+  return new Request(`https://svc/api/prices/historical/${PAST}/${TOKEN_KEY}`)
+}
+
+function batchUrl(timestamps: number[]) {
+  return new Request(
+    `https://svc/api/prices/batchHistorical?coins=${encodeURIComponent(JSON.stringify({ [CHECKSUM_KEY]: timestamps }))}`
+  )
+}
+
+function dbRow(timestamp: number, price: string, token = CHECKSUM) {
+  return {
+    chain: 'ethereum',
+    token,
+    timestamp: new Date(timestamp * 1000),
+    price,
+    symbol: 'WETH',
+    confidence: '0.9',
+    source: 'defillama'
+  }
+}
+
+function resolvingRegistry(price = 27052): HistoricalSourceRegistry {
+  return {
+    resolve: vi.fn(async (_chainId: number, _token: string, timestamp: number) => ({
+      price,
+      timestamp,
+      symbol: 'WETH',
+      confidence: 0.99,
+      source: 'defillama'
+    }))
+  } as unknown as HistoricalSourceRegistry
+}
+
+describe('current UTC day historical lookup', () => {
+  afterEach(() => {
+    vi.useRealTimers()
+    vi.unstubAllGlobals()
+    vi.restoreAllMocks()
+  })
+
+  it('exact: today EOD already in DB is a table hit, no Llama', async () => {
+    vi.useFakeTimers()
+    vi.setSystemTime(NOW * 1000)
+    const fetchMock = vi.fn()
+    vi.stubGlobal('fetch', fetchMock)
+    const queryPool = {
+      query: vi.fn().mockResolvedValue({ rows: [dbRow(TODAY, '123.45')], rowCount: 1 })
+    } as unknown as Pool
+
+    const response = await handleHistorical(exactRequest(), ENV, queryPool, String(NOW - 600), TOKEN_KEY)
+
+    expect(response.status).toBe(200)
+    expect(response.headers.get('cache-control')).toBe(CACHE_CONTROL_TODAY)
+    expect(fetchMock).not.toHaveBeenCalled()
+    await expect(response.json()).resolves.toEqual({
+      coins: {
+        [TOKEN_KEY]: {
+          price: 123.45,
+          symbol: 'WETH',
+          timestamp: TODAY,
+          confidence: 0.9,
+          source: 'defillama'
+        }
+      }
+    })
+  })
+
+  it('exact: today miss calls Llama once, persists today EOD, second request is a table hit', async () => {
+    vi.useFakeTimers()
+    vi.setSystemTime(NOW * 1000)
+    const registry = resolvingRegistry(27052)
+    const stored: unknown[] = []
+    const queryPool = {
+      query: vi.fn(async (sql: string) => {
+        if (String(sql).includes('INSERT INTO token_prices')) {
+          stored.push(dbRow(TODAY, '27052'))
+          return { rows: [], rowCount: 1 }
+        }
+        return { rows: stored, rowCount: stored.length }
+      })
+    } as unknown as Pool
+
+    const first = await handleHistorical(exactRequest(), ENV, queryPool, String(NOW - 600), TOKEN_KEY, registry)
+    expect(first.status).toBe(200)
+    expect(registry.resolve).toHaveBeenCalledTimes(1)
+    await expect(first.json()).resolves.toMatchObject({
+      coins: { [TOKEN_KEY]: { price: 27052, timestamp: TODAY } }
+    })
+    expect(stored).toHaveLength(1)
+
+    const second = await handleHistorical(exactRequest(), ENV, queryPool, String(NOW - 600), TOKEN_KEY, registry)
+    expect(second.status).toBe(200)
+    expect(registry.resolve).toHaveBeenCalledTimes(1)
+    await expect(second.json()).resolves.toMatchObject({
+      coins: { [TOKEN_KEY]: { price: 27052, timestamp: TODAY } }
+    })
+  })
+
+  it('exact: closed-day table hit unchanged', async () => {
+    const fetchMock = vi.fn()
+    vi.stubGlobal('fetch', fetchMock)
+    const response = await handleHistorical(
+      exactRequest(),
+      ENV,
+      { query: vi.fn(async () => ({ rows: [dbRow(PAST, '123.45', RAW)] })) } as unknown as Pool,
+      String(PAST),
+      TOKEN_KEY
+    )
+    expect(response.status).toBe(200)
+    expect(response.headers.get('cache-control')).toBe(CACHE_CONTROL_IMMUTABLE)
+    expect(fetchMock).not.toHaveBeenCalled()
+  })
+
+  it('batch: today EOD already in DB is a table hit, no Llama', async () => {
+    vi.useFakeTimers()
+    vi.setSystemTime(NOW * 1000)
+    const registry = resolvingRegistry(1.5)
+    const queryPool = {
+      query: vi.fn().mockResolvedValue({ rows: [dbRow(TODAY, '1.5')], rowCount: 1 })
+    } as unknown as Pool
+
+    const response = await handleBatchHistorical(batchUrl([TODAY]), ENV, queryPool, registry)
+
+    expect(registry.resolve).not.toHaveBeenCalled()
+    expect(response.headers.get('cache-control')).toBe(CACHE_CONTROL_TODAY)
+    await expect(response.json()).resolves.toEqual({
+      coins: {
+        [CHECKSUM_KEY]: {
+          symbol: 'WETH',
+          prices: [{ timestamp: TODAY, price: 1.5, confidence: 0.9, source: 'defillama' }]
+        }
+      }
+    })
+  })
+
+  it('batch: today miss calls Llama once, persists today EOD, second request is a table hit', async () => {
+    vi.useFakeTimers()
+    vi.setSystemTime(NOW * 1000)
+    const registry = resolvingRegistry(1.5)
+    const stored: unknown[] = []
+    const queryPool = {
+      query: vi.fn(async (sql: string) => {
+        if (String(sql).includes('INSERT INTO token_prices')) {
+          stored.push(dbRow(TODAY, '1.5'))
+          return { rows: [], rowCount: 1 }
+        }
+        return { rows: stored, rowCount: stored.length }
+      })
+    } as unknown as Pool
+
+    const first = await handleBatchHistorical(batchUrl([TODAY]), ENV, queryPool, registry)
+    expect(registry.resolve).toHaveBeenCalledTimes(1)
+    await expect(first.json()).resolves.toEqual({
+      coins: {
+        [CHECKSUM_KEY]: {
+          symbol: 'WETH',
+          prices: [{ timestamp: TODAY, price: 1.5, confidence: 0.99, source: 'defillama' }]
+        }
+      }
+    })
+    expect(stored).toHaveLength(1)
+
+    const second = await handleBatchHistorical(batchUrl([TODAY]), ENV, queryPool, registry)
+    expect(registry.resolve).toHaveBeenCalledTimes(1)
+    await expect(second.json()).resolves.toEqual({
+      coins: {
+        [CHECKSUM_KEY]: {
+          symbol: 'WETH',
+          prices: [{ timestamp: TODAY, price: 1.5, confidence: 0.9, source: 'defillama' }]
+        }
+      }
+    })
+  })
+})

test/prices-batch-range.test.ts

@@ -317,7 +317,7 @@ describe('handleBatchHistorical', () => {
     expect(insertCall).toBeDefined()
   })
 
-  it('does not persist a today-resolved miss', async () => {
+  it('persists a today-resolved miss under the current UTC EOD key', async () => {
     const queryPool = pool([])
 
     const response = await handleBatchHistorical(
@@ -331,7 +331,7 @@ describe('handleBatchHistorical', () => {
     const insertCall = (queryPool.query as ReturnType<typeof vi.fn>).mock.calls.find(([sql]) =>
       String(sql).includes('INSERT INTO token_prices')
     )
-    expect(insertCall).toBeUndefined()
+    expect(insertCall).toBeDefined()
     const body = (await response.json()) as { coins: Record<string, { prices: unknown[] }> }
     expect(body.coins[CHECKSUM_KEY].prices).toHaveLength(1)
   })

test/prices-historical.test.ts

@@ -168,6 +168,22 @@ describe('handleHistorical', () => {
     })
   })
 
+  it('still resolves past days at their normalized end-of-day', async () => {
+    fetchMock.mockResolvedValue(
+      defillamaResponse(200, {
+        coins: {
+          [`ethereum:${RAW_ADDR}`]: { price: 27052, symbol: 'WBTC', timestamp: TIMESTAMP, confidence: 0.99 }
+        }
+      })
+    )
+
+    const response = await handleHistorical(request(), ENV, pool([]), String(TIMESTAMP - 7200), TOKEN_KEY)
+
+    expect(response.status).toBe(200)
+    const url = String(fetchMock.mock.calls[0][0])
+    expect(url).toContain(`/prices/historical/${TIMESTAMP}/`)
+  })
+
   it('resolves current-day requests at now, not the future end-of-day', async () => {
     const now = 1787911200
     vi.useFakeTimers()
@@ -193,9 +209,6 @@ describe('handleHistorical', () => {
     expect(url).toContain(`/prices/historical/${now}/`)
   })
 
-  // The incident's worst hour: right after utc midnight the normalized end-of-day is
-  // ~24h in the future, so DeFiLlama's 6h search window holds no data yet. The mock
-  // answers only for timestamps that have happened — pre-clamp code 404s here.
   it('resolves right after utc midnight, when end-of-day is a day away', async () => {
     const dayStart = 1787875200
     const now = dayStart + 300
@@ -220,20 +233,27 @@ describe('handleHistorical', () => {
     })
   })
 
-  it('still resolves past days at their normalized end-of-day', async () => {
+  it('persists a current-day fallback under the normalized day key', async () => {
+    const now = 1787911200
+    vi.useFakeTimers()
+    vi.setSystemTime(now * 1000)
+
     fetchMock.mockResolvedValue(
       defillamaResponse(200, {
         coins: {
-          [`ethereum:${RAW_ADDR}`]: { price: 27052, symbol: 'WBTC', timestamp: TIMESTAMP, confidence: 0.99 }
+          [`ethereum:${RAW_ADDR}`]: { price: 27052, symbol: 'WBTC', timestamp: now, confidence: 0.99 }
         }
       })
     )
+    const queryPool = pool([])
 
-    const response = await handleHistorical(request(), ENV, pool([]), String(TIMESTAMP - 7200), TOKEN_KEY)
+    const response = await handleHistorical(request(), ENV, queryPool, String(now - 600), TOKEN_KEY)
 
     expect(response.status).toBe(200)
-    const url = String(fetchMock.mock.calls[0][0])
-    expect(url).toContain(`/prices/historical/${TIMESTAMP}/`)
+    const insertCall = (queryPool.query as ReturnType<typeof vi.fn>).mock.calls.find(([sql]) =>
+      String(sql).includes('INSERT INTO token_prices')
+    )
+    expect(insertCall).toBeDefined()
   })
 
   it('persists a resolved fallback under the normalized day key', async () => {
@@ -306,29 +326,6 @@ describe('handleHistorical', () => {
     })
   })
 
-  it('does not persist a current-day fallback', async () => {
-    const now = 1787911200
-    vi.useFakeTimers()
-    vi.setSystemTime(now * 1000)
-
-    fetchMock.mockResolvedValue(
-      defillamaResponse(200, {
-        coins: {
-          [`ethereum:${RAW_ADDR}`]: { price: 27052, symbol: 'WBTC', timestamp: now, confidence: 0.99 }
-        }
-      })
-    )
-    const queryPool = pool([])
-
-    const response = await handleHistorical(request(), ENV, queryPool, String(now - 600), TOKEN_KEY)
-
-    expect(response.status).toBe(200)
-    const insertCall = (queryPool.query as ReturnType<typeof vi.fn>).mock.calls.find(([sql]) =>
-      String(sql).includes('INSERT INTO token_prices')
-    )
-    expect(insertCall).toBeUndefined()
-  })
-
   it('does not persist a future-day fallback', async () => {
     const future = normalizeToEndOfDay(Math.floor(Date.now() / 1000)) + 86_400