diff --git a/CHANGELOG.md b/CHANGELOG.md index 296d2b6f..1710580d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ All notable changes to this project are documented here. The format follows [Kee ### Fixed +- Webull sync imports trade history ([#28](https://github.com/LuxAlgo/trade-journal/issues/28), proposed by @anthony-som in [#29](https://github.com/LuxAlgo/trade-journal/pull/29)). A fill that closes a position opened before Webull's roughly one year of history is skipped, or kept only for the part the history covers, and reported, instead of showing a position that never existed. An option that expired while held closes at 0 at the 4pm New York expiry, and Webull's option contract sizes are saved as multipliers unless you set one. Later syncs only ask for orders from 30 days before the previous sync onward. - Broker sync and IBKR Flex XML uploads keep every fill when an order fills in identical pieces (same symbol, side, size, price and time). Previously the later pieces were dropped as duplicates, so volume, fees and P&L were under-counted and a closed position could stay open, as with a closed Hyperliquid short ([#50](https://github.com/LuxAlgo/trade-journal/issues/50), [#52](https://github.com/LuxAlgo/trade-journal/issues/52)). Fills already stored keep their identity; the next sync adds the missing pieces that are still in the broker's history. IBKR sync now skips order, closed-lot and summary rows in the Flex statement, as file uploads already did, so they are no longer stored as extra fills. - IBKR Flex Query XML statements can be uploaded on Import → File upload, and their times are read in the statement timezone, as broker sync does. Previously an XML upload was not recognized and could not be imported, so the Default import timezone never applied ([#16](https://github.com/LuxAlgo/trade-journal/issues/16)). - Password protection now verifies the session signature on every API route. Previously, when `JOURNAL_PASSWORD` was set, any request carrying a cookie of the right name was accepted, so a forged cookie could read the journal. diff --git a/README.md b/README.md index c2ce370c..65e822a4 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,11 @@ Default import timezone** for Flex timestamps without an offset. Existing synced timezone provenance require recovery into a separate account; see [IBKR timezone recovery](docs/importers.md#ibkr-broker-sync-and-timezone-recovery). +Webull sync reads about a year of order history. A synced fill that closes a position opened before +that is skipped, or kept only for the part the history covers, and the sync reports it, rather than +showing a position that never existed. Options that expired while held close at 0 at the 4pm New +York expiry. + ## Screenshots Captured from the local app with generated demo trades and fictional prop firm records. Expand a view to inspect it at full width. diff --git a/apps/web/package.json b/apps/web/package.json index 49c1c2a6..ef070ed4 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -15,7 +15,7 @@ "@ai-sdk/openai-compatible": "3.0.46", "@dnd-kit/core": "6.3.1", "@dnd-kit/sortable": "10.0.0", - "@luxalgo/broker-sdk": "0.5.1", + "@luxalgo/broker-sdk": "0.6.0", "@luxalgo/journal-core": "workspace:*", "@luxalgo/journal-importers": "workspace:*", "@luxalgo/vela": "^0.6.8", diff --git a/apps/web/src/server/broker-history.ts b/apps/web/src/server/broker-history.ts new file mode 100644 index 00000000..44bdc717 --- /dev/null +++ b/apps/web/src/server/broker-history.ts @@ -0,0 +1,153 @@ +/** + * A broker's trade history starts somewhere (Webull keeps about a year), so a + * sync can see a fill that closes a position opened before that start. Booked + * as is, it shows a position that never existed: selling shares bought before + * the start becomes a short. These pure helpers decide how much of each new + * fill the journal keeps, and close option positions that expired while held. + */ + +export interface SyncedFill { + symbol: string; + side: "buy" | "sell"; + quantity: number; + executedAt: string; + assetClass?: string | null; + /** Broker-stated: whether the fill opened or closed a position. */ + positionEffect?: "open" | "close"; + /** A separately held position on the same symbol, as round trips group it. */ + group?: string; +} + +const EPS = 1e-9; +const direction = (fill: SyncedFill): number => (fill.side === "buy" ? 1 : -1); + +/** + * How much to keep of each `fresh` fill that reaches back before the start of + * the broker's history, by index into `fresh`; 0 drops the fill. Fills kept + * whole are left out. `stored` are fills already journaled; the caller leaves + * out fresh fills that repeat a stored one, since both feed one position. + * + * - A fill the broker labels as a close, with nothing open on that side, is + * dropped. One that closes more than the history opened (sell 3 against 1) + * keeps what the history covers; the rest closed shares bought before the + * start, at a cost the journal never saw. + * - Unlabeled fills of the `checked` asset class, when the broker's position + * list disagrees with where the replay ends: the earliest fill that opened + * the wrong position is cut back, again and again. One that opened from + * flat is dropped; one that crossed flat keeps only its closing part. + * + * Each position (symbol and group) replays on its own, as round trips do. + * + * ponytail: a real flip the broker's list agrees with in direction (sell 3 + * against 2 bought before the start, leaving a short of 1) keeps all 3. + */ +export const keepWithinHistory = ( + stored: SyncedFill[], + fresh: SyncedFill[], + brokerQuantity: (symbol: string) => number | undefined, + checked = "equity", +): Map => { + const keep = new Map(); + const positions = new Map(); + const add = (fill: SyncedFill, index: number) => { + const key = JSON.stringify([fill.symbol, fill.group ?? null]); + const list = positions.get(key); + if (list) list.push({ fill, index }); + else positions.set(key, [{ fill, index }]); + }; + stored.forEach((fill) => add(fill, -1)); + fresh.forEach(add); + + for (const fills of positions.values()) { + fills.sort((a, b) => a.fill.executedAt.localeCompare(b.fill.executedAt)); + const replay = () => { + let held = 0; + // Fresh fills that opened a position, and how much of each closed the one before. + const openers: { index: number; closing: number }[] = []; + for (const { fill, index } of fills) { + const quantity = keep.get(index) ?? fill.quantity; + if (quantity < EPS) continue; + const change = direction(fill) * quantity; + if (index >= 0 && fill.positionEffect === "close") { + if (held * change >= -EPS) { + keep.set(index, 0); + continue; + } + if (quantity > Math.abs(held) + EPS) { + keep.set(index, Math.abs(held)); + held = 0; + continue; + } + } + const after = held + change; + const flips = Math.abs(after) > EPS && Math.sign(after) !== Math.sign(held); + if (index >= 0 && fill.positionEffect === undefined && (Math.abs(held) < EPS || flips)) { + openers.push({ index, closing: Math.abs(held) }); + } + held = after; + } + return { held, openers }; + }; + + let { held, openers } = replay(); + const { symbol, group } = fills[0]!.fill; + // The broker's list nets each symbol, so it only speaks for an ungrouped position. + const expected = group === undefined ? brokerQuantity(symbol) : undefined; + if (expected === undefined || !fills.some(({ fill }) => fill.assetClass === checked)) continue; + while (Math.abs(held - expected) > EPS && Math.sign(held) !== Math.sign(expected)) { + const opener = openers.find(({ index }) => direction(fresh[index]!) === Math.sign(held)); + if (!opener) break; + keep.set(opener.index, opener.closing); + ({ held, openers } = replay()); + } + } + for (const [index, quantity] of keep) { + if (quantity > fresh[index]!.quantity - EPS) keep.delete(index); + } + return keep; +}; + +const OCC_SYMBOL = /^\S+ (\d{2})(\d{2})(\d{2})[CP]\d/; +const newYorkHour = new Intl.DateTimeFormat("en-US", { + timeZone: "America/New_York", + hour: "numeric", + hourCycle: "h23", +}); + +/** 4pm New York, the US options close, on a `YYMMDD` expiry: 20:00Z in summer, 21:00Z in winter. */ +const expiryClose = (yy: string, mm: string, dd: string): string => { + const at = (utcHour: number) => + new Date(Date.UTC(2000 + Number(yy), Number(mm) - 1, Number(dd), utcHour)); + return (newYorkHour.format(at(20)) === "16" ? at(20) : at(21)).toISOString(); +}; + +/** + * Closing fills for option contracts still held after they expired. Webull + * records no fill when an option expires worthless, so the contract would + * otherwise stay open forever and its loss never count. Each closes at 0 at + * the 4pm New York expiry; that fixed time gives it the same hash every sync. + * + * ponytail: a contract exercised or assigned in the money also closes at 0. + * Telling the two apart needs the underlying's price at expiry. + */ +export const expiredOptionCloses = (optionFills: SyncedFill[], now: string): SyncedFill[] => { + const held = new Map(); + for (const fill of optionFills) { + held.set(fill.symbol, (held.get(fill.symbol) ?? 0) + direction(fill) * fill.quantity); + } + const closes: SyncedFill[] = []; + for (const [symbol, quantity] of held) { + const expiry = OCC_SYMBOL.exec(symbol); + if (!expiry || Math.abs(quantity) < EPS) continue; + const executedAt = expiryClose(expiry[1]!, expiry[2]!, expiry[3]!); + if (executedAt >= now) continue; + closes.push({ + symbol, + side: quantity > 0 ? "sell" : "buy", + quantity: Math.abs(quantity), + executedAt, + assetClass: "option", + }); + } + return closes; +}; diff --git a/apps/web/src/server/broker-trades.ts b/apps/web/src/server/broker-trades.ts index 35208c2b..53402b05 100644 --- a/apps/web/src/server/broker-trades.ts +++ b/apps/web/src/server/broker-trades.ts @@ -24,19 +24,33 @@ const numberIdenticalFills = (rows: ImportedExecution[]): ImportedExecution[] => */ export const brokerTradesToExecutions = ( trades: Trade[], -): { timed: ImportedExecution[]; untimed: number } => { - const rows: ImportedExecution[] = trades.map((trade) => ({ - symbol: trade.symbol, - side: trade.side, - quantity: trade.quantity, - price: trade.price, - fee: trade.fee ?? 0, - // The SDK omits unparseable timestamps; a fill with no time can't be - // journaled meaningfully, so it is dropped rather than guessed at. - executedAt: trade.executedAt ?? "", - })); - const timed = rows.filter((row) => row.executedAt !== ""); - return { timed: numberIdenticalFills(timed), untimed: rows.length - timed.length }; +): { + timed: ImportedExecution[]; + untimed: number; + /** Whether each timed row opened or closed a position, where the broker says so. */ + positionEffects: Map; +} => { + // The SDK omits unparseable timestamps; a fill with no time can't be + // journaled meaningfully, so it is dropped rather than guessed at. + const timedTrades = trades.filter((trade) => trade.executedAt); + const timed = numberIdenticalFills( + timedTrades.map((trade) => ({ + symbol: trade.symbol, + side: trade.side, + quantity: trade.quantity, + price: trade.price, + fee: trade.fee ?? 0, + executedAt: trade.executedAt!, + // The SDK's "cash" has no journal equivalent. + ...(trade.assetClass && trade.assetClass !== "cash" ? { assetClass: trade.assetClass } : {}), + })), + ); + const positionEffects = new Map(); + timed.forEach((row, index) => { + const effect = timedTrades[index]!.positionEffect; + if (effect) positionEffects.set(row, effect); + }); + return { timed, untimed: trades.length - timedTrades.length, positionEffects }; }; export const untimedFillsReason = (untimed: number, statementTimeZone: boolean): string => diff --git a/apps/web/src/server/executions.ts b/apps/web/src/server/executions.ts index 20ba400f..a9e3fb2b 100644 --- a/apps/web/src/server/executions.ts +++ b/apps/web/src/server/executions.ts @@ -41,6 +41,11 @@ export const executionProblem = (row: unknown, source: ExecutionSource): string (source === "manual" || !Number.isSafeInteger(r.occurrence) || r.occurrence < 1) ) return `${label}: invalid copy number for an identical fill.`; + if ( + r.hashQuantity !== undefined && + (source !== "sync" || !isFiniteNumber(r.hashQuantity) || r.hashQuantity <= r.quantity) + ) + return `${label}: invalid reported quantity for a fill kept in part.`; const meta = r.importMetadata; const metaOk = !meta || diff --git a/apps/web/src/server/ids.ts b/apps/web/src/server/ids.ts index 954a4186..6a9edc3e 100644 --- a/apps/web/src/server/ids.ts +++ b/apps/web/src/server/ids.ts @@ -7,12 +7,14 @@ export const nowIso = (): string => new Date().toISOString(); /** * Content hash for execution dedup: the same fill inserts once per account. * Numbered copies of an identical fill hash apart; the first copy keeps the - * plain hash, so fills stored before numbering existed still deduplicate. + * plain hash, so fills stored before numbering existed still deduplicate. A + * fill kept only in part hashes with the quantity the broker reported. */ export const executionHash = (execution: { symbol: string; side: string; quantity: number; + hashQuantity?: number; price: number; executedAt: string; importMetadata?: { id: string; group?: string }; @@ -23,7 +25,7 @@ export const executionHash = (execution: { [ execution.symbol, execution.side, - execution.quantity.toPrecision(12), + (execution.hashQuantity ?? execution.quantity).toPrecision(12), execution.price.toPrecision(12), execution.executedAt, ...(execution.importMetadata diff --git a/apps/web/src/server/sync.ts b/apps/web/src/server/sync.ts index 0fad6aeb..b6b8b759 100644 --- a/apps/web/src/server/sync.ts +++ b/apps/web/src/server/sync.ts @@ -1,11 +1,13 @@ import { connect, listBrokers, type BrokerId } from "@luxalgo/broker-sdk"; -import { eq } from "drizzle-orm"; -import { accounts, db } from "@/db"; +import { and, eq } from "drizzle-orm"; +import type { ImportedExecution } from "@luxalgo/journal-importers"; +import { accounts, db, executions } from "@/db"; import { decryptJson, encryptJson } from "./crypto"; -import { nowIso } from "./ids"; +import { executionHash, nowIso } from "./ids"; import { brokerTradesToExecutions, untimedFillsReason } from "./broker-trades"; +import { expiredOptionCloses, keepWithinHistory, type SyncedFill } from "./broker-history"; import { insertExecutions, type InsertResult } from "./executions"; -import { getImportTimeZone } from "./settings"; +import { getImportTimeZone, getMultipliers, setSetting } from "./settings"; import { requireValue } from "./api"; import { assertIbkrSyncTimeZone, @@ -17,6 +19,11 @@ import { withoutSummaryRows } from "./ibkr-flex-xml-import"; /** All broker connectivity goes through @luxalgo/broker-sdk, never direct API code. */ export { listBrokers }; +/** The asset class whose fills a broker's complete position list can vouch for. */ +const CHECKED_ASSET_CLASS: Record = { webull: "equity" }; +/** Later syncs re-read this much before the last one, for orders that filled after placement. */ +const HISTORY_OVERLAP_MS = 30 * 86_400_000; + /** * The SDK reads every Flex `` row as a fill. Keep execution rows only, * as file uploads do, so order and summary rows are not journaled as fills. @@ -50,9 +57,21 @@ export const syncAccount = async (accountId: string): Promise => { if (statementTimeZone !== undefined) assertIbkrSyncTimeZone(account, statementTimeZone); const credentials = decryptJson>(account.credentialsEnc); + // Brokers that page through trade history (Webull) only need recent fills + // once history has landed; refetched fills dedupe by content hash. + const hasSyncedFills = db + .select({ id: executions.id }) + .from(executions) + .where(and(eq(executions.accountId, accountId), eq(executions.source, "sync"))) + .get(); + const historySince = + hasSyncedFills && account.lastSyncAt + ? new Date(Date.parse(account.lastSyncAt) - HISTORY_OVERLAP_MS).toISOString() + : undefined; const connection = connect({ broker: account.broker as BrokerId, credentials, + ...(historySince !== undefined ? { historySince } : {}), ...(statementTimeZone !== undefined ? { statementTimeZone } : {}), ...(isIbkrSyncAccount(account) ? { fetch: flexExecutionRowsOnly } : {}), // Some brokers rotate tokens on every fetch (Questrade): persist or die. @@ -67,9 +86,63 @@ export const syncAccount = async (accountId: string): Promise => { const snapshot = await connection.fetchSnapshot(); const syncedAt = nowIso(); - const { timed, untimed } = brokerTradesToExecutions( - snapshot.accounts.flatMap((brokerAccount) => brokerAccount.trades), + const trades = snapshot.accounts.flatMap((brokerAccount) => brokerAccount.trades); + // Option P&L needs the contract size; the broker's value never replaces the user's. + const multipliers = getMultipliers(); + const stated = trades.filter( + (trade) => trade.multiplier && multipliers[trade.symbol] === undefined, ); + for (const trade of stated) multipliers[trade.symbol] = trade.multiplier!; + if (stated.length > 0) setSetting("multipliers", JSON.stringify(multipliers)); + const { timed, untimed, positionEffects } = brokerTradesToExecutions(trades); + + // A broker whose position list is complete for an asset class can show + // which unlabeled fills reach back before the start of its history. + const checked = CHECKED_ASSET_CLASS[account.broker ?? ""]; + const brokerQuantities = new Map(); + for (const position of snapshot.accounts.flatMap((a) => a.positions)) { + brokerQuantities.set( + position.symbol, + (brokerQuantities.get(position.symbol) ?? 0) + position.quantity, + ); + } + const brokerQuantity = (symbol: string) => + checked ? (brokerQuantities.get(symbol) ?? 0) : undefined; + let dropped = 0; + let trimmed = 0; + const withinHistory = (fills: ImportedExecution[]): ImportedExecution[] => { + const stored = db.select().from(executions).where(eq(executions.accountId, accountId)).all(); + const storedHashes = new Set(stored.map((row) => row.contentHash)); + const fresh = fills.filter((fill) => !storedHashes.has(executionHash(fill))); + const groupOf = (json: string | null) => + json ? (JSON.parse(json) as { group?: string }).group : undefined; + const keep = keepWithinHistory( + stored.map((row) => ({ + ...row, + side: row.side as "buy" | "sell", + group: groupOf(row.importMetadataJson), + })), + fresh.map((fill) => ({ + ...fill, + group: fill.importMetadata?.group, + positionEffect: positionEffects.get(fill), + })), + brokerQuantity, + checked, + ); + const kept = new Map([...keep].map(([index, quantity]) => [fresh[index]!, quantity])); + dropped = [...keep.values()].filter((quantity) => quantity < 1e-9).length; + trimmed = keep.size - dropped; + // Repeats of stored fills go through, so the insert counts them as duplicates. + return fills.flatMap((fill) => { + const quantity = kept.get(fill); + if (quantity === undefined) return [fill]; + if (quantity < 1e-9) return []; + // The kept part hashes as the broker's whole fill, so later syncs still match it. + const fee = (fill.fee * quantity) / fill.quantity; + return [{ ...fill, quantity, fee, hashQuantity: fill.quantity }]; + }); + }; const equity = snapshot.accounts.reduce((total, a) => total + a.equity, 0); const positions = snapshot.accounts.flatMap((a) => a.positions); @@ -86,7 +159,26 @@ export const syncAccount = async (accountId: string): Promise => { assertIbkrSyncTimeZone(current, statementTimeZone); } // Validation, history provenance, deduplication and sync status commit together. - const inserted = insertExecutions(accountId, timed, "sync"); + const inserted = insertExecutions(accountId, withinHistory(timed), "sync"); + if (account.broker === "webull") { + const optionFills = db + .select() + .from(executions) + .where(and(eq(executions.accountId, accountId), eq(executions.assetClass, "option"))) + .all() + .map((row) => ({ ...row, side: row.side as "buy" | "sell" })); + const expired = expiredOptionCloses(optionFills, syncedAt).map((fill) => ({ + ...fill, + assetClass: "option" as const, + price: 0, + fee: 0, + })); + // An expiry has no commission, so default fee rules must not add one. + if (expired.length > 0) + inserted.inserted += insertExecutions(accountId, expired, "sync", undefined, { + preserveFees: true, + }).inserted; + } db.update(accounts) .set({ lastSyncAt: syncedAt, @@ -101,6 +193,20 @@ export const syncAccount = async (accountId: string): Promise => { }, { behavior: "immediate" }, ); + if (dropped > 0) { + result.skipped += dropped; + if (result.skippedReasons.length < 5) + result.skippedReasons.push( + `${dropped} fill(s) closed a position opened before the broker's available history and were skipped.`, + ); + } + if (trimmed > 0) { + result.skipped += trimmed; + if (result.skippedReasons.length < 5) + result.skippedReasons.push( + `${trimmed} fill(s) also closed a position opened before the broker's available history; only the part that history covers was kept.`, + ); + } if (untimed > 0) { result.skipped += untimed; if (result.skippedReasons.length < 5) diff --git a/apps/web/tests/broker-history.test.ts b/apps/web/tests/broker-history.test.ts new file mode 100644 index 00000000..3ca8f078 --- /dev/null +++ b/apps/web/tests/broker-history.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "vitest"; +import { expiredOptionCloses, keepWithinHistory, type SyncedFill } from "@/server/broker-history"; + +const stock = (side: "buy" | "sell", quantity: number, day: number): SyncedFill => ({ + symbol: "ABC", + side, + quantity, + executedAt: `2026-10-${String(day).padStart(2, "0")}T15:00:00.000Z`, + assetClass: "equity", +}); +const option = ( + side: "buy" | "sell", + quantity: number, + day: number, + positionEffect: "open" | "close", +): SyncedFill => ({ + ...stock(side, quantity, day), + symbol: "XYZ 261218C50", + assetClass: "option", + positionEffect, +}); +const flat = () => 0; +const kept = (map: Map) => Object.fromEntries(map); + +describe("fills that reach back before the broker's history", () => { + it("a sale of shares bought before the history is dropped rather than booked as a short", () => { + expect(kept(keepWithinHistory([], [stock("sell", 7, 1)], flat))).toEqual({ 0: 0 }); + }); + + it("a short the broker still reports is kept", () => { + expect(keepWithinHistory([], [stock("sell", 7, 1)], () => -7).size).toBe(0); + }); + + it("a position opened and closed inside the history is kept", () => { + expect(keepWithinHistory([], [stock("sell", 7, 1), stock("buy", 7, 2)], flat).size).toBe(0); + }); + + it("selling more than the history bought keeps the part the history covers", () => { + const fresh = [ + stock("buy", 1, 1), + stock("sell", 3, 2), + stock("buy", 1, 5), + stock("sell", 1, 6), + ]; + expect(kept(keepWithinHistory([], fresh, flat))).toEqual({ 1: 1 }); + }); + + it("the same holds when the buy was journaled by an earlier sync", () => { + expect(kept(keepWithinHistory([stock("buy", 1, 1)], [stock("sell", 3, 2)], flat))).toEqual({ + 0: 1, + }); + }); + + it("a flip the broker still shows is kept whole", () => { + expect(keepWithinHistory([], [stock("buy", 1, 1), stock("sell", 2, 2)], () => -1).size).toBe(0); + }); + + it("a labeled close with nothing open is dropped, and one closing more than was opened is cut back", () => { + const fresh = [ + option("sell", 2, 1, "close"), + option("buy", 1, 2, "open"), + option("sell", 3, 3, "close"), + ]; + expect(kept(keepWithinHistory([], fresh, () => undefined))).toEqual({ 0: 0, 2: 1 }); + }); + + it("a labeled close against an open journaled earlier is kept", () => { + const stored = [option("buy", 1, 1, "open")]; + expect(keepWithinHistory(stored, [option("sell", 1, 2, "close")], () => undefined).size).toBe( + 0, + ); + }); + + it("a broker without a complete position list keeps every unlabeled fill", () => { + expect(keepWithinHistory([], [stock("sell", 7, 1)], () => undefined).size).toBe(0); + }); + + it("separately held positions on one symbol replay apart", () => { + const long = ( + side: "buy" | "sell", + quantity: number, + day: number, + effect: "open" | "close", + ) => ({ + ...stock(side, quantity, day), + group: "long", + positionEffect: effect, + }); + const short = ( + side: "buy" | "sell", + quantity: number, + day: number, + effect: "open" | "close", + ) => ({ + ...stock(side, quantity, day), + group: "short", + positionEffect: effect, + }); + // Covering the short must not read as closing the long, or the other way round. + const fresh = [ + long("buy", 1, 1, "open"), + short("sell", 2, 2, "open"), + long("sell", 1, 3, "close"), + ]; + expect(keepWithinHistory([], fresh, flat).size).toBe(0); + }); +}); + +describe("options that expire while held", () => { + const now = "2026-10-26T12:00:00.000Z"; + const held = (symbol: string, side: "buy" | "sell", day: string): SyncedFill => ({ + symbol, + side, + quantity: 2, + executedAt: `${day}T15:00:00.000Z`, + assetClass: "option", + }); + + it("close at 0 at the 4pm New York expiry, in summer and in winter", () => { + expect( + expiredOptionCloses( + [held("AAA 260717C60", "buy", "2026-07-01"), held("BBB 251219P20", "sell", "2025-12-01")], + now, + ), + ).toEqual([ + { + symbol: "AAA 260717C60", + side: "sell", + quantity: 2, + executedAt: "2026-07-17T20:00:00.000Z", + assetClass: "option", + }, + { + symbol: "BBB 251219P20", + side: "buy", + quantity: 2, + executedAt: "2025-12-19T21:00:00.000Z", + assetClass: "option", + }, + ]); + }); + + it("leave alone contracts already closed or not yet expired", () => { + const closed = [ + held("CCC 260918C10", "buy", "2026-09-01"), + held("CCC 260918C10", "sell", "2026-09-02"), + ]; + expect( + expiredOptionCloses([...closed, held("DDD 261218C10", "buy", "2026-10-01")], now), + ).toEqual([]); + }); +}); diff --git a/apps/web/tests/webull-sync-flow.test.ts b/apps/web/tests/webull-sync-flow.test.ts new file mode 100644 index 00000000..2d7823b7 --- /dev/null +++ b/apps/web/tests/webull-sync-flow.test.ts @@ -0,0 +1,175 @@ +import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const scratch = mkdtempSync(join(tmpdir(), "journal-webull-sync-")); +vi.stubEnv("JOURNAL_DATA_DIR", scratch); +const { db, accounts, executions, settings, trades } = await import("../src/db"); +const { encryptJson } = await import("../src/server/crypto"); +const { syncAccount } = await import("../src/server/sync"); + +const DAY_MS = 86_400_000; +const daysAgo = (days: number) => new Date(Date.now() - days * DAY_MS); + +// Webull historical orders, as broker-sdk's Webull adapter reads them. +const order = (fields: Record) => ({ + status: "FILLED", + instrument_type: "EQUITY", + fees: [], + commission: {}, + ...fields, +}); +let orders: Record[] = []; +let positions: Record[] = []; + +beforeEach(() => { + db.delete(trades).run(); + db.delete(executions).run(); + db.delete(accounts).run(); + db.delete(settings).run(); + db.insert(accounts) + .values({ + id: "wb", + name: "Webull", + kind: "sync", + broker: "webull", + credentialsEnc: encryptJson({ apiKey: "k", apiSecret: "s" }), + createdAt: "2026-01-01", + }) + .run(); + orders = []; + positions = []; + vi.stubGlobal( + "fetch", + vi.fn(async (input) => { + const url = new URL(String(input)); + if (url.pathname === "/trading/accounts/list") return Response.json([{ account_id: "A1" }]); + if (url.pathname === "/trading/assets/balances/get") + return Response.json({ total_net_liquidation_value: "1000" }); + if (url.pathname === "/trading/assets/positions/list") return Response.json(positions); + if (url.pathname === "/trading/orders/historical-orders/list") { + // Each 30-day window answers with the orders filled inside it. + const start = Date.parse(url.searchParams.get("start_time")!); + const end = Date.parse(url.searchParams.get("end_time")!); + return Response.json({ + data: orders.filter((o) => { + const at = Date.parse(String(o.filled_time_at)); + return at >= start && at < end; + }), + }); + } + throw new Error(`Unexpected Webull request: ${url.pathname}`); + }), + ); +}); +afterEach(() => vi.unstubAllGlobals()); +afterAll(() => { + db.$client.close(); + vi.unstubAllEnvs(); + rmSync(scratch, { recursive: true, force: true }); +}); + +describe("Webull sync", () => { + it("selling shares bought before the history floor is skipped, not journaled as a short", async () => { + orders = [ + order({ + symbol: "ABC", + side: "SELL", + filled_quantity: "7", + filled_price: "20", + filled_time_at: daysAgo(3).toISOString(), + }), + ]; + const outcome = await syncAccount("wb"); + expect(outcome).toMatchObject({ inserted: 0, skipped: 1 }); + expect(outcome.skippedReasons.join(" ")).toContain("before the broker's available history"); + expect(db.select().from(trades).all()).toEqual([]); + }); + + it("a close of shares partly bought before the history keeps the covered part, and later syncs leave it be", async () => { + // 2 shares bought before the floor, 1 more bought after the first sync, then all 3 sold. + const buy = order({ + symbol: "ABC", + side: "BUY", + filled_quantity: "1", + filled_price: "100", + filled_time_at: daysAgo(5).toISOString(), + }); + orders = [buy]; + positions = [{ symbol: "ABC", quantity: "3", market_value: "300" }]; + expect(await syncAccount("wb")).toMatchObject({ inserted: 1, skipped: 0 }); + + orders = [ + buy, + order({ + symbol: "ABC", + side: "SELL", + filled_quantity: "3", + filled_price: "110", + filled_time_at: daysAgo(1).toISOString(), + }), + ]; + positions = []; + const outcome = await syncAccount("wb"); + expect(outcome).toMatchObject({ inserted: 1, skipped: 1 }); + expect(outcome.skippedReasons.join(" ")).toContain( + "only the part that history covers was kept", + ); + const [trade] = db.select().from(trades).all(); + expect(trade).toMatchObject({ status: "win", direction: "long", quantity: 1 }); + expect(trade!.grossPnl).toBeCloseTo(10, 9); + + expect(await syncAccount("wb")).toMatchObject({ inserted: 0, duplicates: 2, skipped: 0 }); + }); + + it("two identical orders filled in the same second are both kept, and a re-sync adds nothing", async () => { + const filledAt = daysAgo(2).toISOString(); + const buy = { + symbol: "ABC", + side: "BUY", + filled_quantity: "1", + filled_price: "100", + filled_time_at: filledAt, + }; + orders = [order(buy), order(buy)]; + positions = [{ symbol: "ABC", quantity: "2", market_value: "200" }]; + + expect(await syncAccount("wb")).toMatchObject({ inserted: 2, skipped: 0 }); + expect(await syncAccount("wb")).toMatchObject({ inserted: 0, duplicates: 2, skipped: 0 }); + expect(db.select().from(trades).all()).toMatchObject([ + { symbol: "ABC", status: "open", openQuantity: 2 }, + ]); + }); + + it("an option that expired while held closes at 0, priced with its contract size", async () => { + const expiry = daysAgo(10).toISOString().slice(0, 10); + orders = [ + order({ + symbol: "XYZ", + side: "BUY", + instrument_type: "OPTION", + position_intent: "BUY_TO_OPEN", + legs: [ + { + symbol: "XYZ", + option_type: "PUT", + option_expire_date: expiry, + strike_price: "50", + option_contract_multiplier: "100", + }, + ], + filled_quantity: "1", + filled_price: "1.10", + filled_time_at: daysAgo(20).toISOString(), + }), + ]; + + expect(await syncAccount("wb")).toMatchObject({ inserted: 2 }); + const [trade] = db.select().from(trades).all(); + expect(trade).toMatchObject({ status: "loss", direction: "long" }); + expect(trade!.grossPnl).toBeCloseTo(-110, 9); + // The next sync recomputes nothing: the expiry close is already journaled. + expect(await syncAccount("wb")).toMatchObject({ inserted: 0, duplicates: 1 }); + }); +}); diff --git a/packages/importers/src/types.ts b/packages/importers/src/types.ts index a95b83ea..c0b11640 100644 --- a/packages/importers/src/types.ts +++ b/packages/importers/src/types.ts @@ -18,6 +18,12 @@ export interface ImportedExecution { * price and time are each kept. The first copy has none. */ occurrence?: number; + /** + * The quantity a broker reported for a fill the journal keeps only in part, + * because the rest closed a position opened before the broker's history. + * The content hash uses it, so the fill still deduplicates on later syncs. + */ + hashQuantity?: number; /** Untrusted export labels are resolved to saved source IDs by the import review. */ ninjaTrader?: { sourceKey: string; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2e19c57b..79fc9809 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -42,8 +42,8 @@ importers: specifier: 10.0.0 version: 10.0.0(@dnd-kit/core@6.3.1(react-dom@19.2.8(react@19.2.8))(react@19.2.8))(react@19.2.8) '@luxalgo/broker-sdk': - specifier: 0.5.1 - version: 0.5.1(react@19.2.8) + specifier: 0.6.0 + version: 0.6.0(react@19.2.8) '@luxalgo/journal-core': specifier: workspace:* version: link:../../packages/core @@ -617,8 +617,8 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} - '@luxalgo/broker-sdk@0.5.1': - resolution: {integrity: sha512-JS+q5kBK6mBFviL3h9q+/R0KEBzddKOtELUGG1KvlyHt/W4Fba/5zz9i9MPyLXLyc5HtNFVxwUra+4wrZa+nfQ==} + '@luxalgo/broker-sdk@0.6.0': + resolution: {integrity: sha512-DmsrHPDrFqRaTeZ+x/yAf5/RfEu62aUNt67nIGlKOynRgn3aEOR/0qQBM+qaFITeL/pY4PCtQqCWp/TcGo8QDA==} engines: {node: '>=18.17'} hasBin: true peerDependencies: @@ -3209,7 +3209,7 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@luxalgo/broker-sdk@0.5.1(react@19.2.8)': + '@luxalgo/broker-sdk@0.6.0(react@19.2.8)': optionalDependencies: react: 19.2.8