From cdae8b50167689ddfa30fbadf887a94ea7ecbbae Mon Sep 17 00:00:00 2001 From: zahei <33495369+zahei@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:39:40 -0500 Subject: [PATCH 1/2] feat(importers): read XTB xStation .xlsx account statements XTB exports one workbook per account with Closed Positions, Open Positions and Cash Operations sheets. Closed rows become an entry and an exit fill; open lots become entry fills tagged with their XTB position id so a later workbook that closes the position adds the exit. Cash operations are skipped. Includes a dependency-free .xlsx reader (ZIP + worksheet XML). Contributed in #42. --- apps/web/src/app/import/page.tsx | 21 +- packages/importers/src/detect.ts | 2 + packages/importers/src/formats/xtb.ts | 326 ++++++++++++++++++++++++++ packages/importers/src/index.ts | 1 + packages/importers/src/xlsx.ts | 232 ++++++++++++++++++ 5 files changed, 577 insertions(+), 5 deletions(-) create mode 100644 packages/importers/src/formats/xtb.ts create mode 100644 packages/importers/src/xlsx.ts diff --git a/apps/web/src/app/import/page.tsx b/apps/web/src/app/import/page.tsx index d0d11705..f13539ec 100644 --- a/apps/web/src/app/import/page.tsx +++ b/apps/web/src/app/import/page.tsx @@ -21,6 +21,7 @@ import { import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { postJson, useApi } from "@/lib/use-api"; import { decodeImportFile } from "@/lib/decode-import"; +import { xtbWorkbookToStatement } from "@luxalgo/journal-importers"; import { formatTimestamp, isTimeZone } from "@/lib/timezone"; import { dayKeyOf } from "@luxalgo/journal-core"; import { ImportReconciliation } from "@/components/import-reconciliation"; @@ -193,16 +194,23 @@ function FileImport() { setBusy(true); try { const pdf = /\.pdf$/i.test(file.name); + const xlsx = /\.xlsx$/i.test(file.name); if (pdf && !aiEnabled) throw new Error("Enable AI parsing to upload a PDF statement."); + if (xlsx && aiEnabled) + throw new Error( + "Turn off AI parsing for an XTB Excel report. The XTB importer reads it directly.", + ); if (aiEnabled && file.size > AI_IMPORT_MAX_BYTES) throw new Error("AI uploads must be 8 MB or smaller."); if (aiEnabled && !/\.(csv|tsv|txt|html?|xml|pdf)$/i.test(file.name)) throw new Error( - "Choose a CSV, TSV, HTML, XML, TXT or PDF file. Export spreadsheets as CSV first.", + "Choose a CSV, TSV, HTML, XML, TXT or PDF file. XTB Excel reports use the XTB importer with AI turned off.", ); const buffer = await file.arrayBuffer(); let text: string; - if (pdf) { + if (xlsx) { + text = await xtbWorkbookToStatement(buffer); + } else if (pdf) { const bytes = new Uint8Array(buffer); let binary = ""; for (let i = 0; i < bytes.length; i += 8192) @@ -395,7 +403,7 @@ function FileImport() { {fileName || (aiEnabled ? "Choose a statement for AI parsing" - : "Choose a CSV / HTML / XML statement")} + : "Choose a CSV / HTML / XML statement or an XTB Excel report")} {aiEnabled ? ( @@ -408,14 +416,17 @@ function FileImport() { formatData?.formats.map((format) => format.label.split(" (")[0]) ?? [], ), ].join(", ")}{" "} - — anything else via column mapping. + — anything else via column mapping. XTB: one .xlsx per account. Closed positions + and open holdings are imported; cash operations are left out. )} + /[",\n\r]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value; + +const sheetCsv = (rows: string[][]): string => + rows.map((row) => row.map((cell) => csvField(cell)).join(",")).join("\n"); + +const headerIndex = (rows: string[][]): number => + rows.findIndex((row) => hasHeaders(row, CLOSED_HEADERS)); + +const isClosedSheet = (sheet: XlsxSheet): boolean => + /closed positions/i.test(sheet.name) || headerIndex(sheet.rows) !== -1; + +const openHeaderIndex = (rows: string[][]): number => + rows.findIndex((row) => hasHeaders(row, OPEN_HEADERS) && !hasHeaders(row, [["closeprice"]])); + +const isOpenSheet = (sheet: XlsxSheet): boolean => + /open positions/i.test(sheet.name) || + (openHeaderIndex(sheet.rows) !== -1 && !isClosedSheet(sheet)); + +const countCashOperations = (sheet: XlsxSheet | undefined): number => { + if (!sheet) return 0; + const header = sheet.rows.findIndex((row) => hasHeaders(row, [["type"], ["amount"], ["time"]])); + if (header === -1) return 0; + const records = toRecords(sheet.rows.slice(header)); + return records.filter((row) => { + const type = (pick(row, ["type"]) ?? "").trim(); + return type !== "" && !/^total$/i.test(type); + }).length; +}; + +const countOpenLots = (sheet: XlsxSheet | undefined): number => { + if (!sheet) return 0; + const header = openHeaderIndex(sheet.rows); + if (header === -1) return 0; + return toRecords(sheet.rows.slice(header)).filter((row) => + /^(buy|sell)$/i.test(pick(row, ["type"]) ?? ""), + ).length; +}; + +/** + * Reduce one XTB account workbook to one statement: closed positions, then + * open holdings. Cash operations are counted in a note and are not executions. + * One workbook is one account. + */ +export const xtbWorkbookToStatement = async (data: ArrayBuffer | Uint8Array): Promise => { + const sheets = await readXlsx(data); + const closed = sheets.find(isClosedSheet); + if (!closed) throw new Error("This XTB workbook has no Closed Positions sheet."); + const open = sheets.find(isOpenSheet); + const openCount = countOpenLots(open); + const cashCount = countCashOperations( + sheets.find((sheet) => /cash operations/i.test(sheet.name)), + ); + const note = [ + NOTE, + `Imported ${openCount} open holding(s) as entry fills. Skipped ${cashCount} cash operation(s). Import each XTB account workbook into its own journal account.`, + ]; + const parts = [sheetCsv(closed.rows)]; + if (open) parts.push([SECTION, "open"].join(","), sheetCsv(open.rows)); + parts.push(note.map(csvField).join(",")); + return parts.join("\n"); +}; + +const assetClassOf = (category: string | undefined): AssetClass | undefined => { + const text = (category ?? "").toLowerCase(); + if (/etf|stock|equit|share/.test(text)) return "equity"; + if (/forex|\bfx\b/.test(text)) return "forex"; + if (/cfd/.test(text)) return "cfd"; + if (/crypto/.test(text)) return "crypto"; + if (/future/.test(text)) return "futures"; + if (/option/.test(text)) return "option"; + return undefined; +}; + +const accountFrom = (rows: string[][]): string | undefined => { + for (const row of rows) { + if (headerKey(row[0] ?? "") === "accountnumber" && row[1]?.trim()) return row[1].trim(); + } + return undefined; +}; + +const positionGroup = ( + account: string | undefined, + positionId: string, + symbol: string, + openedAt: string, +) => `xtb:${account ?? "account"}:${positionId || `${symbol}:${openedAt}`}`; + +const splitSections = (table: string[][]): { closed: string[][]; open: string[][] } => { + const marker = table.findIndex((row) => row[0] === SECTION); + if (marker === -1) return { closed: table, open: [] }; + return { closed: table.slice(0, marker), open: table.slice(marker + 1) }; +}; + +/** + * XTB xStation account history. Closed positions become an entry and an exit. + * Open holdings become the entry only, tagged with the same XTB position id, so + * a later closed-position report adds the exit and skips the entry already + * stored. Prices stay in the instrument currency. Gross Profit on a close is + * the account-currency result and is stored as reported P&L. Unrealized profit + * on an open holding is not a journal result. + */ +export const xtb: ImportFormat = { + id: "xtb", + label: "XTB (account statement)", + detect: (_headers, content) => headerIndex(parseCsv(content)) !== -1, + parse: (content, _options: ImportOptions): ParsedImport => { + const table = parseCsv(content); + const { closed, open } = splitSections(table); + const headerAt = headerIndex(closed); + const records = headerAt === -1 ? [] : toRecords(closed.slice(headerAt)); + const account = accountFrom(closed.slice(0, Math.max(headerAt, 0))); + const executions: ImportedExecution[] = []; + const warnings = [ + "XTB closed positions are trade-level. Each row becomes an entry and an exit at the reported prices. Account-currency Gross Profit is kept as the position result, because price × volume is not the P&L when a conversion rate applies.", + "Open holdings are entry fills tagged with their XTB position id. A later report that closes the same id adds the exit and skips this entry.", + "Open Time and Close Time are labeled UTC and are imported as UTC.", + ]; + const errors: string[] = []; + let skippedRows = 0; + const accounts = new Set(); + if (account) accounts.add(account); + const entryGroups = new Set(); + + const note = table.find((row) => row[0] === NOTE)?.[1]; + if (note) warnings.push(note); + + for (const row of records) { + if (row[Object.keys(row)[0] ?? ""] === NOTE || Object.values(row).includes(NOTE)) continue; + const parsed = closedRow(row, account, accounts); + if (parsed === "skip") { + if (Object.values(row).some((value) => value !== "")) skippedRows++; + continue; + } + if (parsed === "invalid") { + skippedRows++; + continue; + } + entryGroups.add(parsed.entry.importMetadata!.group!); + executions.push(parsed.entry, parsed.exit); + } + + const openHeader = openHeaderIndex(open); + const openAccount = + accountFrom(openHeader === -1 ? open : open.slice(0, openHeader)) ?? account; + if (openAccount) accounts.add(openAccount); + const openRecords = openHeader === -1 ? [] : toRecords(open.slice(openHeader)); + for (const row of openRecords) { + const holding = openRow(row, openAccount, accounts); + if (holding === "skip") { + if (Object.values(row).some((value) => value !== "")) skippedRows++; + continue; + } + if (holding === "invalid") { + skippedRows++; + continue; + } + if (entryGroups.has(holding.importMetadata!.group!)) continue; + entryGroups.add(holding.importMetadata!.group!); + executions.push(holding); + } + + if (accounts.size > 1) + errors.push( + "This file contains more than one XTB account. Import one account workbook at a time.", + ); + if (executions.length === 0) + errors.push("This XTB report has no closed or open positions to import."); + + return { + format: "xtb", + executions, + skippedRows, + warnings, + ...(errors.length ? { errors } : {}), + }; + }, +}; + +const closedRow = ( + row: Row, + account: string | undefined, + accounts: Set, +): { entry: ImportedExecution; exit: ImportedExecution } | "skip" | "invalid" => { + const type = (pick(row, ["type"]) ?? "").trim().toLowerCase(); + if (type !== "buy" && type !== "sell") return "skip"; + const symbol = pick(row, ["ticker", "symbol"])?.trim().toUpperCase(); + const quantity = parseQuantity(pick(row, ["volume"])); + const entryPrice = parseMoney(pick(row, ["openprice"])); + const exitPrice = parseMoney(pick(row, ["closeprice"])); + const openedAt = parseTimestamp(pick(row, ["opentimeutc", "opentime"]), "UTC"); + const closedAt = parseTimestamp(pick(row, ["closetimeutc", "closetime"]), "UTC"); + const positionId = pick(row, ["positionid"])?.trim() ?? ""; + const rowAccount = pick(row, ["account", "accountnumber"])?.trim() || account; + if (rowAccount) accounts.add(rowAccount); + if ( + !symbol || + !openedAt || + !closedAt || + !Number.isFinite(quantity) || + quantity <= 0 || + !Number.isFinite(entryPrice) || + !Number.isFinite(exitPrice) + ) + return "invalid"; + + const gross = parseMoney(pick(row, ["grossprofit"])); + const net = parseMoney(pick(row, ["profitloss"])); + let reportedGrossPnl: number | undefined; + let fees = 0; + if (Number.isFinite(gross) && Number.isFinite(net)) { + const cost = gross - net; + if (cost >= 0) { + reportedGrossPnl = gross; + fees = cost; + } else { + reportedGrossPnl = net; + } + } else if (Number.isFinite(gross)) { + reportedGrossPnl = gross; + } else if (Number.isFinite(net)) { + reportedGrossPnl = net; + } + + const trade: ImportedTrade = { + symbol, + direction: type === "buy" ? "long" : "short", + quantity, + entryPrice, + exitPrice, + openedAt, + closedAt, + fees, + assetClass: assetClassOf(pick(row, ["category"])), + }; + const [entry, exit] = tradeToExecutions(trade); + const group = positionGroup(rowAccount, positionId, symbol, openedAt); + entry.importMetadata = { id: "entry", group, order: 0, preserveFee: true }; + exit.importMetadata = { + id: "exit", + group, + order: 1, + preserveFee: true, + ...(reportedGrossPnl !== undefined ? { reportedGrossPnl } : {}), + }; + return { entry, exit }; +}; + +const openRow = ( + row: Row, + account: string | undefined, + accounts: Set, +): ImportedExecution | "skip" | "invalid" => { + const type = (pick(row, ["type"]) ?? "").trim().toLowerCase(); + if (type !== "buy" && type !== "sell") return "skip"; + const symbol = pick(row, ["ticker"])?.trim().toUpperCase(); + const quantity = parseQuantity(pick(row, ["volume"])); + const price = parseMoney(pick(row, ["openprice"])); + const openedAt = parseTimestamp(pick(row, ["opentimeutc", "opentime"]), "UTC"); + const positionId = (pick(row, ["positionid", "instrumentposition"]) ?? "").trim(); + const rowAccount = pick(row, ["account", "accountnumber"])?.trim() || account; + if (rowAccount) accounts.add(rowAccount); + if ( + !symbol || + !openedAt || + !/^\d+$/.test(positionId) || + !Number.isFinite(quantity) || + quantity <= 0 || + !Number.isFinite(price) + ) + return "invalid"; + const commission = parseMoney(pick(row, ["opencommission"])); + return { + symbol, + side: type === "buy" ? "buy" : "sell", + quantity, + price, + fee: Number.isFinite(commission) ? Math.abs(commission) : 0, + executedAt: openedAt, + assetClass: assetClassOf(pick(row, ["category"])), + importMetadata: { + id: "entry", + group: positionGroup(rowAccount, positionId, symbol, openedAt), + order: 0, + preserveFee: true, + }, + }; +}; diff --git a/packages/importers/src/index.ts b/packages/importers/src/index.ts index cf40d03d..ab2e0db1 100644 --- a/packages/importers/src/index.ts +++ b/packages/importers/src/index.ts @@ -6,4 +6,5 @@ export { parseMoney, parseQuantity } from "./numbers"; export { parseWithMapping, readHeaders, type GenericMapping } from "./formats/generic"; export { makeFillsFormat, parseSide, type FillsFormatSpec } from "./formats/fills"; export { parseHistory, type HistoryParseOptions } from "./formats/history"; +export { xtbWorkbookToStatement } from "./formats/xtb"; export { positionFillProblem } from "./position-fills"; diff --git a/packages/importers/src/xlsx.ts b/packages/importers/src/xlsx.ts new file mode 100644 index 00000000..8b89df47 --- /dev/null +++ b/packages/importers/src/xlsx.ts @@ -0,0 +1,232 @@ +/** Minimal .xlsx reader (ZIP + worksheet XML). No dependency; runs in Node and the browser. */ + +export interface XlsxSheet { + name: string; + rows: string[][]; +} + +const EXCEL_EPOCH_MS = Date.UTC(1899, 11, 30); +const BUILTIN_DATE_FORMATS = new Set([ + 14, 15, 16, 17, 18, 19, 20, 21, 22, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 45, 46, 47, 50, 51, + 52, 53, 54, 55, 56, 57, 58, +]); + +const u16 = (bytes: Uint8Array, offset: number): number => + bytes[offset]! | (bytes[offset + 1]! << 8); + +const u32 = (bytes: Uint8Array, offset: number): number => + (bytes[offset]! | + (bytes[offset + 1]! << 8) | + (bytes[offset + 2]! << 16) | + (bytes[offset + 3]! << 24)) >>> + 0; + +const decodeXml = (value: string): string => + value + .replace(/&#(\d+);/g, (_, digits: string) => String.fromCodePoint(Number(digits))) + .replace(/&#x([0-9a-f]+);/gi, (_, hex: string) => String.fromCodePoint(parseInt(hex, 16))) + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&/g, "&"); + +const formatLooksLikeDate = (code: string): boolean => { + const stripped = code.replace(/"[^"]*"/g, "").replace(/\[[^\]]*\]/g, ""); + return /[ydhms]/i.test(stripped); +}; + +const excelSerialToIso = (serial: number, epochMs: number): string => + new Date(Math.round(epochMs + serial * 86_400_000)).toISOString(); + +const formatNumber = (value: number): string => { + if (!Number.isFinite(value)) return ""; + const nearest = Math.round(value); + // Position and account ids are large integers; Excel stores them as floats. + if (Math.abs(value) >= 1e6 && Math.abs(value - nearest) < 1e-3) return String(nearest); + const text = Object.is(value, -0) ? "0" : String(value); + if (!/[eE]/.test(text)) return text; + return value.toFixed(12).replace(/\.?0+$/, ""); +}; + +const columnIndex = (cellRef: string): number => { + const letters = /^[A-Z]+/i.exec(cellRef)?.[0] ?? "A"; + let index = 0; + for (const char of letters.toUpperCase()) index = index * 26 + (char.charCodeAt(0) - 64); + return index - 1; +}; + +const rowNumber = (cellRef: string): number => Number(/\d+$/.exec(cellRef)?.[0] ?? 0); + +interface ZipEntry { + name: string; + method: number; + compressedSize: number; + dataOffset: number; +} + +const findEocd = (bytes: Uint8Array): number => { + const start = Math.max(0, bytes.length - 22 - 65_535); + for (let offset = bytes.length - 22; offset >= start; offset--) { + if ( + bytes[offset] === 0x50 && + bytes[offset + 1] === 0x4b && + bytes[offset + 2] === 0x05 && + bytes[offset + 3] === 0x06 + ) + return offset; + } + throw new Error("This file is not an .xlsx workbook."); +}; + +const zipEntries = (bytes: Uint8Array): ZipEntry[] => { + const eocd = findEocd(bytes); + const count = u16(bytes, eocd + 10); + let cursor = u32(bytes, eocd + 16); + const entries: ZipEntry[] = []; + for (let index = 0; index < count; index++) { + if (u32(bytes, cursor) !== 0x02014b50) throw new Error("This file is not an .xlsx workbook."); + const method = u16(bytes, cursor + 10); + const compressedSize = u32(bytes, cursor + 20); + const nameLength = u16(bytes, cursor + 28); + const extraLength = u16(bytes, cursor + 30); + const commentLength = u16(bytes, cursor + 32); + const localOffset = u32(bytes, cursor + 42); + const name = new TextDecoder().decode(bytes.subarray(cursor + 46, cursor + 46 + nameLength)); + const localNameLength = u16(bytes, localOffset + 26); + const localExtraLength = u16(bytes, localOffset + 28); + entries.push({ + name, + method, + compressedSize, + dataOffset: localOffset + 30 + localNameLength + localExtraLength, + }); + cursor += 46 + nameLength + extraLength + commentLength; + } + return entries; +}; + +const inflateRaw = async (data: Uint8Array): Promise => { + if (data.length === 0) return new Uint8Array(); + const stream = new Blob([data.slice()]) + .stream() + .pipeThrough(new DecompressionStream("deflate-raw")); + return new Uint8Array(await new Response(stream).arrayBuffer()); +}; + +const readEntry = async (bytes: Uint8Array, entry: ZipEntry): Promise => { + const compressed = bytes.subarray(entry.dataOffset, entry.dataOffset + entry.compressedSize); + if (entry.method === 0) return new TextDecoder().decode(compressed); + if (entry.method === 8) return new TextDecoder().decode(await inflateRaw(compressed)); + throw new Error(`Unsupported Excel compression (method ${entry.method}).`); +}; + +const sharedStrings = (xml: string): string[] => + [...xml.matchAll(/]*>([\s\S]*?)<\/si>/g)].map((match) => + [...match[1]!.matchAll(/]*>([\s\S]*?)<\/t>/g)] + .map((text) => decodeXml(text[1]!)) + .join(""), + ); + +const dateStyleIndexes = (xml: string): Set => { + const custom = new Map(); + for (const match of xml.matchAll(/]*numFmtId="(\d+)"[^>]*formatCode="([^"]*)"/g)) + custom.set(Number(match[1]), decodeXml(match[2]!)); + const block = xml.match(/]*>([\s\S]*?)<\/cellXfs>/)?.[1] ?? ""; + const indexes = new Set(); + let style = 0; + for (const match of block.matchAll(/]*numFmtId="(\d+)"/g)) { + const id = Number(match[1]); + const code = custom.get(id); + if (BUILTIN_DATE_FORMATS.has(id) || (code !== undefined && formatLooksLikeDate(code))) + indexes.add(style); + style++; + } + return indexes; +}; + +const sheetRows = ( + xml: string, + strings: string[], + dates: Set, + epochMs: number, +): string[][] => { + const cells = new Map>(); + let maxRow = 0; + let maxColumn = 0; + for (const match of xml.matchAll(/]*?)(?:\/>|>([\s\S]*?)<\/c>)/g)) { + const attrs = match[1] ?? ""; + const body = match[2] ?? ""; + const ref = / r="([^"]+)"/.exec(attrs)?.[1]; + if (!ref) continue; + const row = rowNumber(ref); + const column = columnIndex(ref); + const type = / t="([^"]+)"/.exec(attrs)?.[1]; + const style = Number(/ s="(\d+)"/.exec(attrs)?.[1] ?? 0); + const raw = /]*>([\s\S]*?)<\/v>/.exec(body)?.[1]; + const inline = [...body.matchAll(/]*>([\s\S]*?)<\/t>/g)] + .map((text) => decodeXml(text[1]!)) + .join(""); + let value = ""; + if (type === "s" && raw !== undefined) value = strings[Number(raw)] ?? ""; + else if (type === "inlineStr" || type === "str") value = inline || (raw ? decodeXml(raw) : ""); + else if (raw !== undefined && raw !== "") { + const number = Number(raw); + value = + dates.has(style) && Number.isFinite(number) + ? excelSerialToIso(number, epochMs) + : formatNumber(number); + } else value = inline; + if (!cells.has(row)) cells.set(row, new Map()); + cells.get(row)!.set(column, value); + maxRow = Math.max(maxRow, row); + maxColumn = Math.max(maxColumn, column); + } + const rows: string[][] = []; + for (let row = 1; row <= maxRow; row++) { + const line = cells.get(row); + if (!line || [...line.values()].every((value) => value === "")) continue; + const cellsInRow = Array.from({ length: maxColumn + 1 }, (_, index) => line.get(index) ?? ""); + while (cellsInRow.length > 0 && cellsInRow[cellsInRow.length - 1] === "") cellsInRow.pop(); + rows.push(cellsInRow); + } + return rows; +}; + +/** Read every worksheet. Dates become UTC ISO strings; other numbers stay plain decimals. */ +export const readXlsx = async (data: ArrayBuffer | Uint8Array): Promise => { + const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); + if (bytes.length < 4 || bytes[0] !== 0x50 || bytes[1] !== 0x4b) + throw new Error("This file is not an .xlsx workbook."); + const entries = zipEntries(bytes); + const byName = new Map(entries.map((entry) => [entry.name.replace(/\\/g, "/"), entry])); + const read = async (name: string): Promise => { + const entry = byName.get(name) ?? byName.get(`xl/${name.replace(/^xl\//, "")}`); + return entry ? readEntry(bytes, entry) : null; + }; + const workbook = await read("xl/workbook.xml"); + const rels = await read("xl/_rels/workbook.xml.rels"); + if (!workbook || !rels) throw new Error("This file is not an .xlsx workbook."); + const strings = sharedStrings((await read("xl/sharedStrings.xml")) ?? ""); + const dates = dateStyleIndexes((await read("xl/styles.xml")) ?? ""); + const epochMs = /date1904="(?:1|true)"/.test(workbook) ? Date.UTC(1904, 0, 1) : EXCEL_EPOCH_MS; + const targets = new Map(); + for (const match of rels.matchAll(/]*)\/?>/g)) { + const id = /\bId="([^"]+)"/.exec(match[1]!)?.[1]; + const target = /\bTarget="([^"]+)"/.exec(match[1]!)?.[1]; + if (id && target) targets.set(id, target); + } + const sheets: XlsxSheet[] = []; + for (const match of workbook.matchAll(/]*)\/?>/g)) { + const name = /\bname="([^"]+)"/.exec(match[1]!)?.[1]; + const id = /\br:id="([^"]+)"/.exec(match[1]!)?.[1]; + const target = id ? targets.get(id) : undefined; + if (!name || !target) continue; + const path = target.startsWith("/") ? target.slice(1) : `xl/${target.replace(/^\//, "")}`; + const xml = await read(path); + if (!xml) continue; + sheets.push({ name: decodeXml(name), rows: sheetRows(xml, strings, dates, epochMs) }); + } + if (sheets.length === 0) throw new Error("This workbook has no sheets."); + return sheets; +}; From 07cd06de973aa21a4947c4ec5568ce372bf582b8 Mon Sep 17 00:00:00 2001 From: Christopher Downie Date: Tue, 13 Oct 2026 17:58:11 -0500 Subject: [PATCH 2/2] fix(xtb): harden the XTB .xlsx importer for release Review fixes on top of the contributed importer (#42): - Typecheck: give the importers package DOM lib typings for TextDecoder and DecompressionStream, and type tradeToExecutions as an entry/exit pair - Decode workbooks on the server: the page uploads the .xlsx as base64 with encoding "xlsx" (5 MB cap), so preview and commit parse the same server-side text and the reader stays out of the client bundle - Bound the ZIP reader: check every offset, refuse ZIP64, multi-part, encrypted members and legacy .xls with a clear message, cap entries and expanded size, and stop inflating past a member's declared size - Refuse a later close whose lot differs from the stored holding (for example a partial close) instead of booking a phantom open position; tolerate sub-second and rounding differences for the same lot - Open lots inherit the category of their ticker's rollup row, so holdings keep their asset class when a later workbook closes them - Count only unknown trade types as skipped; totals, rollups and notes are layout rows - Replace the real-export fixtures with synthetic workbooks generated in the layout of xStation exports; no account data is committed Fixes #42 --- CHANGELOG.md | 1 + README.md | 32 +- apps/web/src/app/api/import/route.ts | 50 +++- apps/web/src/app/import/page.tsx | 16 +- apps/web/src/lib/xlsx-import.ts | 2 + apps/web/src/server/xtb-import.ts | 91 ++++++ apps/web/tests/xtb-import-ui.test.ts | 110 +++++++ apps/web/tests/xtb-import.test.ts | 174 +++++++++++ docs/importers.md | 33 +++ packages/importers/src/formats/xtb.ts | 22 +- packages/importers/src/types.ts | 2 +- packages/importers/src/xlsx.ts | 115 ++++++-- packages/importers/tests/xtb-fixture.ts | 361 +++++++++++++++++++++++ packages/importers/tests/xtb.test.ts | 373 ++++++++++++++++++++++++ packages/importers/tsconfig.json | 4 + 15 files changed, 1315 insertions(+), 71 deletions(-) create mode 100644 apps/web/src/lib/xlsx-import.ts create mode 100644 apps/web/src/server/xtb-import.ts create mode 100644 apps/web/tests/xtb-import-ui.test.ts create mode 100644 apps/web/tests/xtb-import.test.ts create mode 100644 packages/importers/tests/xtb-fixture.ts create mode 100644 packages/importers/tests/xtb.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d5150f48..71665f77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ All notable changes to this project are documented here. The format follows [Kee - PDF and PNG review exports; JSON export now includes folders, templates, rule assessments, routines and defaults - Configurable breakeven tolerance, default fees, and default stop/target distances per account and symbol - Importers: MetaTrader 5 deal reports and TradingView strategy exports +- Importers: XTB xStation `.xlsx` account workbooks, with closed positions, open holdings and later closes of those holdings (contributed by @zahei, [#42](https://github.com/LuxAlgo/trade-journal/issues/42)) - Realized R accounts for contract multipliers and scaled entries - Optional historical market data connections (London Strategic Edge, Alpaca, OANDA, Binance, Coinbase, or your own candle CSV) with estimated MAE/MFE and candle replay on closed trades - Prop firm tracker: evaluation and reset costs, refunds, payout requests, receipts, cash ROI, renewals, attachments, CSV import and export diff --git a/README.md b/README.md index caba9e34..3a9f2e90 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ One primitive drives everything: a raw **execution** (a fill). Executions come i ```mermaid flowchart LR BS["Broker sync (broker-sdk)"] --> EX["Executions (deduped fills)"] - IM["15 documented statement formats + CSV mapper"] --> EX + IM["16 documented statement formats + CSV mapper"] --> EX MN["Manual entry"] --> EX EX --> RT["Round-trip engine (FIFO / LIFO / WAVG)"] RT --> D["Dashboard + calendar"] @@ -126,21 +126,21 @@ flowchart LR ## Features -| | | -| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Broker sync** | Read-only sync via [`@luxalgo/broker-sdk`](https://github.com/LuxAlgo/broker-sdk): Alpaca, Binance, Bybit, Coinbase, Kraken, OKX, Tradier, IBKR Flex, Hyperliquid, Questrade, Topstep, Trading212, Webull, Crypto.com, E*TRADE, Public, Charles Schwab, TradeStation, tastytrade, Robinhood Crypto, Gemini, and KuCoin. Connect forms render straight from SDK metadata. | -| **Statement import** | **15 documented formats**, auto-detected: **TradeZella** and **Tradervue** (CSV migration), TradingView (paper and strategy exports), MetaTrader 4 and 5, ThinkorSwim, IBKR (activity + Flex), NinjaTrader, Tradovate, TopstepX, Webull, DAS Trader, plus a column mapper for any other CSV. Matching execution records dedupe within an account; review warnings and skipped rows before committing an import. Have an export we don't recognize? Open an issue with an anonymized sample; real files are the most useful contribution this repo can get. | -| **Round-trip engine** | Flat-to-flat position cycles from raw fills. FIFO / LIFO / weighted-average per account. Partial fills, scale-ins, flips, futures multipliers. Annotations survive rebuilds. | -| **Analytics** | Net/gross P&L, win and day-win rates, profit factor, expectancy, R multiples, streaks, drawdown and recovery, profit concentration, duration/time-of-day/weekday performance, per-symbol/tag/mistake/playbook breakdowns. Advanced filters on every dimension, comparison groups, and a two-way cross-analysis matrix. Reports also include rolling performance trends and a trade explorer scatter plot, with saved MAE/MFE estimates where available. | -| **Dashboard** | P&L calendar with weekly totals, cumulative and daily P&L, gauges, the open **Edge Score v2** radar, open positions, time-of-day performance. Drag cards to rearrange, hide what you don't use, save named layouts. Calendar insights highlight daily patterns. Light/dark themes, a collapsible sidebar, and mobile navigation support smaller screens. | -| **Trade pages** | Charted on [Vela](https://www.npmjs.com/package/@luxalgo/vela) with entry/exit markers and P&L labels: fill paths from recorded executions, optional user-selected market candles, and trade replay. Estimated MAE/MFE, running P&L, executions, ratings, stops/targets, tags, mistakes. | -| **Daily journal** | Day stats, intraday P&L curve, autosaving Markdown notes with templates and attachments (images, PDFs). Type them or **dictate** them with browser speech recognition (no journal API key required; browser support and speech processing vary). | -| **Notebook & playbooks** | Folders, search, tags, trade links; named setups with rule checklists scored per trade, with adherence and followed-vs-broken performance in Reports. | -| **AI reflection** | Bring your own Anthropic or OpenAI API key: session recaps, per-trade critiques, "ask your journal" over your own aggregates. Key encrypted at rest; requests go from your server to the model, nowhere else. | -| **Routines & misses** | Pre-, during- and post-session routines with weekday schedules and a 13-week history; a missed-opportunity log kept out of your trading metrics. | -| **Journal defaults** | Configure breakeven tolerance, plus fee and stop/target rules per account and symbol; set timezone and contract multipliers for consistent calculations. | -| **Prop firms** | Evaluation and reset expenses, refunds, payout requests, partial receipts, reversals, cash ROI, account phases, renewal reminders, attachments, and generic cash CSV import/export. Separate from trade P&L. | -| **Privacy & export** | Privacy mode masks every monetary value (charts keep their shape) and persists across tabs. Trades export to CSV, reviews to PDF or PNG, and journal records to JSON; credentials, candle datasets, and attachment binaries are excluded from that export. | +| | | +| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Broker sync** | Read-only sync via [`@luxalgo/broker-sdk`](https://github.com/LuxAlgo/broker-sdk): Alpaca, Binance, Bybit, Coinbase, Kraken, OKX, Tradier, IBKR Flex, Hyperliquid, Questrade, Topstep, Trading212, Webull, Crypto.com, E*TRADE, Public, Charles Schwab, TradeStation, tastytrade, Robinhood Crypto, Gemini, and KuCoin. Connect forms render straight from SDK metadata. | +| **Statement import** | **16 documented formats**, auto-detected: **TradeZella** and **Tradervue** (CSV migration), TradingView (paper and strategy exports), MetaTrader 4 and 5, ThinkorSwim, IBKR (activity + Flex), NinjaTrader, Tradovate, TopstepX, Webull, DAS Trader, XTB, plus a column mapper for any other CSV. Matching execution records dedupe within an account; review warnings and skipped rows before committing an import. Have an export we don't recognize? Open an issue with an anonymized sample; real files are the most useful contribution this repo can get. | +| **Round-trip engine** | Flat-to-flat position cycles from raw fills. FIFO / LIFO / weighted-average per account. Partial fills, scale-ins, flips, futures multipliers. Annotations survive rebuilds. | +| **Analytics** | Net/gross P&L, win and day-win rates, profit factor, expectancy, R multiples, streaks, drawdown and recovery, profit concentration, duration/time-of-day/weekday performance, per-symbol/tag/mistake/playbook breakdowns. Advanced filters on every dimension, comparison groups, and a two-way cross-analysis matrix. Reports also include rolling performance trends and a trade explorer scatter plot, with saved MAE/MFE estimates where available. | +| **Dashboard** | P&L calendar with weekly totals, cumulative and daily P&L, gauges, the open **Edge Score v2** radar, open positions, time-of-day performance. Drag cards to rearrange, hide what you don't use, save named layouts. Calendar insights highlight daily patterns. Light/dark themes, a collapsible sidebar, and mobile navigation support smaller screens. | +| **Trade pages** | Charted on [Vela](https://www.npmjs.com/package/@luxalgo/vela) with entry/exit markers and P&L labels: fill paths from recorded executions, optional user-selected market candles, and trade replay. Estimated MAE/MFE, running P&L, executions, ratings, stops/targets, tags, mistakes. | +| **Daily journal** | Day stats, intraday P&L curve, autosaving Markdown notes with templates and attachments (images, PDFs). Type them or **dictate** them with browser speech recognition (no journal API key required; browser support and speech processing vary). | +| **Notebook & playbooks** | Folders, search, tags, trade links; named setups with rule checklists scored per trade, with adherence and followed-vs-broken performance in Reports. | +| **AI reflection** | Bring your own Anthropic or OpenAI API key: session recaps, per-trade critiques, "ask your journal" over your own aggregates. Key encrypted at rest; requests go from your server to the model, nowhere else. | +| **Routines & misses** | Pre-, during- and post-session routines with weekday schedules and a 13-week history; a missed-opportunity log kept out of your trading metrics. | +| **Journal defaults** | Configure breakeven tolerance, plus fee and stop/target rules per account and symbol; set timezone and contract multipliers for consistent calculations. | +| **Prop firms** | Evaluation and reset expenses, refunds, payout requests, partial receipts, reversals, cash ROI, account phases, renewal reminders, attachments, and generic cash CSV import/export. Separate from trade P&L. | +| **Privacy & export** | Privacy mode masks every monetary value (charts keep their shape) and persists across tabs. Trades export to CSV, reviews to PDF or PNG, and journal records to JSON; credentials, candle datasets, and attachment binaries are excluded from that export. | IBKR broker sync and uploaded Flex XML statements use **Settings → General → Default import timezone** for Flex timestamps without an offset. Existing synced accounts with unknown or different diff --git a/apps/web/src/app/api/import/route.ts b/apps/web/src/app/api/import/route.ts index d698ee54..ca554959 100644 --- a/apps/web/src/app/api/import/route.ts +++ b/apps/web/src/app/api/import/route.ts @@ -18,6 +18,7 @@ import { looksLikeFlexXml, parseFlexXmlImport, } from "@/server/ibkr-flex-xml-import"; +import { reconcileXtbHoldings, xlsxStatement } from "@/server/xtb-import"; import type { AiImportOptions } from "@/lib/ai-import"; import { createAiImportPreview, @@ -36,7 +37,7 @@ interface ImportBody { symbol?: string; review?: ImportReviewOptions; ai?: AiImportOptions; - encoding?: "text" | "pdf"; + encoding?: "text" | "pdf" | "xlsx"; aiPreviewToken?: string; aiReviewed?: boolean; } @@ -58,15 +59,22 @@ export const POST = handler(async (request: Request) => { requireValue(isTimeZone(body.timeZone), "Enter a valid IANA statement timezone."); const timeZone = body.timeZone ?? getImportTimeZone(); requireValue( - body.encoding === undefined || body.encoding === "text" || body.encoding === "pdf", + body.encoding === undefined || ["text", "pdf", "xlsx"].includes(body.encoding), "Unsupported file encoding.", ); requireValue(body.encoding !== "pdf" || body.ai, "Enable AI parsing to upload PDF statements."); + requireValue( + body.encoding !== "xlsx" || (!body.ai && !body.mapping), + "Turn off AI parsing for an XTB Excel report. The XTB importer reads it directly.", + ); requireValue(!body.ai || !body.mapping, "Choose AI parsing or column mapping, not both."); + // Workbooks are reduced to a statement here, so preview and commit parse the same server-side text. + const content = body.encoding === "xlsx" ? await xlsxStatement(body.content) : body.content; const statement = { content: body.content, fileName: body.fileName, - encoding: body.encoding, + // Only AI parsing reads this, and a workbook with AI parsing is refused above. + encoding: body.encoding === "xlsx" ? undefined : body.encoding, timeZone, }; if (body.ai && body.mode === "commit") @@ -80,16 +88,16 @@ export const POST = handler(async (request: Request) => { ? await parseStatementWithAi(statement, body.ai, request.signal) : readAiImportPreview(statement, body.aiPreviewToken!) : body.mapping - ? parseWithMapping(body.content, body.mapping, { timeZone }) - : looksLikeFlexXml(body.content) - ? parseFlexXmlImport(body.content, timeZone) - : parseAuto(body.content, { timeZone, fileName: body.fileName, symbol: body.symbol }); + ? parseWithMapping(content, body.mapping, { timeZone }) + : looksLikeFlexXml(content) + ? parseFlexXmlImport(content, timeZone) + : parseAuto(content, { timeZone, fileName: body.fileName, symbol: body.symbol }); if (!parsed) { return ok({ detected: null, timeZone, - headers: readHeaders(body.content), + headers: readHeaders(content), needsMapping: true, }); } @@ -98,7 +106,18 @@ export const POST = handler(async (request: Request) => { parsed.format === IBKR_FLEX_XML.id && body.accountId ? flexUploadSyncProblem(body.accountId, timeZone) : null; - const errors = [...(parsed.errors ?? []), ...(syncProblem ? [syncProblem] : [])]; + const xtb = + parsed.format === "xtb" && body.accountId + ? reconcileXtbHoldings(body.accountId, parsed.executions as ImportedExecution[]) + : null; + const problems = [syncProblem, xtb?.problem].filter((problem): problem is string => !!problem); + const errors = [...(parsed.errors ?? []), ...problems]; + const warnings = [ + ...(parsed.warnings ?? []), + ...(xtb?.matched + ? [`${xtb.matched} open holding(s) imported earlier are closed by this file.`] + : []), + ]; if (body.mode === "preview") { const symbols = [...new Set(parsed.executions.map((e) => e.symbol))]; @@ -126,8 +145,8 @@ export const POST = handler(async (request: Request) => { null, ), }, - warnings: parsed.warnings, - errors: syncProblem ? errors : parsed.errors, + warnings, + errors: problems.length ? errors : parsed.errors, needsSymbol: parsed.needsSymbol, reconciliation: parsed.format === "ninjatrader" && body.accountId @@ -144,21 +163,22 @@ export const POST = handler(async (request: Request) => { ? commitNinjaTraderImport(body.accountId, parsed, body.content, timeZone, body.review) : insertExecutions( body.accountId, - parsed.executions as ImportedExecution[], + xtb?.executions ?? (parsed.executions as ImportedExecution[]), "import", undefined, { preserveFees: Boolean(body.ai) }, ); + if (xtb?.matched) result.duplicates += xtb.matched; // Invalid rows are skipped with a warning rather than failing the whole file. - const warnings = [ - ...(parsed.warnings ?? []), + const commitWarnings = [ + ...warnings, ...(result.skipped > 0 ? [ `${result.skipped} row(s) were skipped because they could not be journaled: ${result.skippedReasons.join(" ")}`, ] : []), ]; - return ok({ detected: parsed.format, ...result, warnings }); + return ok({ detected: parsed.format, ...result, warnings: commitWarnings }); }); /** The import page lists what auto-detection understands. */ diff --git a/apps/web/src/app/import/page.tsx b/apps/web/src/app/import/page.tsx index f13539ec..e1bfb9e0 100644 --- a/apps/web/src/app/import/page.tsx +++ b/apps/web/src/app/import/page.tsx @@ -21,7 +21,6 @@ import { import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { postJson, useApi } from "@/lib/use-api"; import { decodeImportFile } from "@/lib/decode-import"; -import { xtbWorkbookToStatement } from "@luxalgo/journal-importers"; import { formatTimestamp, isTimeZone } from "@/lib/timezone"; import { dayKeyOf } from "@luxalgo/journal-core"; import { ImportReconciliation } from "@/components/import-reconciliation"; @@ -34,6 +33,7 @@ import { AI_IMPORT_MAX_TEXT, type AiImportOptions as AiOptions, } from "@/lib/ai-import"; +import { XLSX_MAX_BYTES } from "@/lib/xlsx-import"; import { Checkbox } from "@/components/ui/checkbox"; interface BrokerInfo { @@ -156,7 +156,7 @@ function FileImport() { model: AI_DEFAULT_MODELS.openai, apiKey: "", }); - const [encoding, setEncoding] = useState<"text" | "pdf">("text"); + const [encoding, setEncoding] = useState<"text" | "pdf" | "xlsx">("text"); const [aiReviewed, setAiReviewed] = useState(false); const { data: formatData } = useApi<{ formats: { id: string; label: string }[] }>("/api/import"); const { data: settingsData, error: settingsError } = useApi< @@ -202,15 +202,16 @@ function FileImport() { ); if (aiEnabled && file.size > AI_IMPORT_MAX_BYTES) throw new Error("AI uploads must be 8 MB or smaller."); + if (xlsx && file.size > XLSX_MAX_BYTES) + throw new Error("Excel uploads must be a .xlsx file of 5 MB or less."); if (aiEnabled && !/\.(csv|tsv|txt|html?|xml|pdf)$/i.test(file.name)) throw new Error( "Choose a CSV, TSV, HTML, XML, TXT or PDF file. XTB Excel reports use the XTB importer with AI turned off.", ); const buffer = await file.arrayBuffer(); let text: string; - if (xlsx) { - text = await xtbWorkbookToStatement(buffer); - } else if (pdf) { + // Binary statements travel as base64; the server reads them. + if (pdf || xlsx) { const bytes = new Uint8Array(buffer); let binary = ""; for (let i = 0; i < bytes.length; i += 8192) @@ -223,7 +224,8 @@ function FileImport() { "Use a smaller export: AI text parsing supports up to 150,000 characters.", ); } - setEncoding(pdf ? "pdf" : "text"); + const fileEncoding = pdf ? "pdf" : xlsx ? "xlsx" : "text"; + setEncoding(fileEncoding); setContent(text); // Choosing a file alone never sends it to an AI provider. if (aiEnabled) return; @@ -235,6 +237,7 @@ function FileImport() { review: {}, fileName: file.name, timeZone, + ...(fileEncoding === "xlsx" ? { encoding: fileEncoding } : {}), }), ); } catch (cause) { @@ -312,6 +315,7 @@ function FileImport() { symbol, // Commit with the exact parsing zone used by the reviewed preview. timeZone: preview.timeZone, + ...(encoding === "xlsx" ? { encoding } : {}), ...(preview.aiPreviewToken ? { ai: { provider: aiOptions.provider, model: aiOptions.model }, diff --git a/apps/web/src/lib/xlsx-import.ts b/apps/web/src/lib/xlsx-import.ts new file mode 100644 index 00000000..5499409f --- /dev/null +++ b/apps/web/src/lib/xlsx-import.ts @@ -0,0 +1,2 @@ +/** Largest .xlsx statement the import page accepts; XTB account workbooks are far smaller. */ +export const XLSX_MAX_BYTES = 5 * 1024 * 1024; diff --git a/apps/web/src/server/xtb-import.ts b/apps/web/src/server/xtb-import.ts new file mode 100644 index 00000000..27f183e3 --- /dev/null +++ b/apps/web/src/server/xtb-import.ts @@ -0,0 +1,91 @@ +import { and, eq } from "drizzle-orm"; +import { xtbWorkbookToStatement, type ImportedExecution } from "@luxalgo/journal-importers"; +import { db, executions } from "@/db"; +import { requireValue } from "./api"; +import { XLSX_MAX_BYTES } from "@/lib/xlsx-import"; +import { executionHash } from "./ids"; + +/** Decode an uploaded base64 .xlsx on the server; the browser never parses the workbook. */ +export const xlsxStatement = async (base64: string): Promise => { + const tooLarge = "Excel uploads must be a .xlsx file of 5 MB or less."; + requireValue(base64.length <= Math.ceil(XLSX_MAX_BYTES / 3) * 4, tooLarge); + requireValue(/^[A-Za-z0-9+/]+={0,2}$/.test(base64), "This file is not an .xlsx workbook."); + const bytes = Buffer.from(base64, "base64"); + requireValue(bytes.length <= XLSX_MAX_BYTES, tooLarge); + try { + return await xtbWorkbookToStatement(bytes); + } catch (error) { + // Reader and XTB layout errors describe the file, so they are the user's answer. + if (error instanceof Error) requireValue(false, error.message); + throw error; + } +}; + +const isXtbEntry = (fill: ImportedExecution): boolean => + fill.importMetadata?.id === "entry" && Boolean(fill.importMetadata.group?.startsWith("xtb:")); + +/** The same lot read from Open Positions and later from Closed Positions. */ +const sameLot = (stored: ImportedExecution, incoming: ImportedExecution): boolean => + stored.symbol === incoming.symbol && + stored.side === incoming.side && + stored.quantity.toPrecision(12) === incoming.quantity.toPrecision(12) && + Math.abs(Date.parse(stored.executedAt) - Date.parse(incoming.executedAt)) < 1000 && + Math.abs(stored.price - incoming.price) <= + 1e-3 * Math.max(Math.abs(stored.price), Math.abs(incoming.price)); + +const describe = (fill: ImportedExecution) => + `${fill.side} ${fill.quantity} ${fill.symbol} at ${fill.price} on ${fill.executedAt}`; + +/** + * An XTB holding is stored as an entry fill keyed by its position id. When a + * later workbook closes that position, its entry must be the stored one, or the + * journal would keep a second, phantom open lot. A lot whose time or price + * differ only by rounding is the same fill and is not inserted again; a lot + * whose size or side changed (for example a partial close) is refused, because + * which part was closed cannot be known. Nothing is saved in that case. + */ +export const reconcileXtbHoldings = ( + accountId: string, + incoming: ImportedExecution[], +): { executions: ImportedExecution[]; matched: number; problem: string | null } => { + const entries = incoming.filter(isXtbEntry); + if (!entries.length) return { executions: incoming, matched: 0, problem: null }; + const stored = new Map(); + for (const row of db + .select() + .from(executions) + .where(and(eq(executions.accountId, accountId), eq(executions.source, "import"))) + .all()) { + const importMetadata = row.importMetadataJson ? JSON.parse(row.importMetadataJson) : undefined; + const fill: ImportedExecution = { + symbol: row.symbol, + side: row.side as "buy" | "sell", + quantity: row.quantity, + price: row.price, + fee: row.fee, + executedAt: row.executedAt, + importMetadata, + }; + if (isXtbEntry(fill)) stored.set(importMetadata.group, { fill, hash: row.contentHash }); + } + + const dropped = new Set(); + for (const entry of entries) { + const previous = stored.get(entry.importMetadata!.group!); + if (!previous || previous.hash === executionHash(entry)) continue; + if (!sameLot(previous.fill, entry)) { + const position = entry.importMetadata!.group!.split(":").at(-1); + return { + executions: incoming, + matched: 0, + problem: `XTB position ${position} was imported earlier as ${describe(previous.fill)}, but this file reports ${describe(entry)}. The position changed, for example by a partial close, so the journal cannot tell which part this file closes. Nothing was saved. Export one workbook whose date range covers both the opening and the close of this position, and import it into a new journal account.`, + }; + } + dropped.add(entry); + } + return { + executions: incoming.filter((fill) => !dropped.has(fill)), + matched: dropped.size, + problem: null, + }; +}; diff --git a/apps/web/tests/xtb-import-ui.test.ts b/apps/web/tests/xtb-import-ui.test.ts new file mode 100644 index 00000000..09349eab --- /dev/null +++ b/apps/web/tests/xtb-import-ui.test.ts @@ -0,0 +1,110 @@ +// @vitest-environment jsdom +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { TooltipProvider } from "../src/components/ui/tooltip"; +const state = vi.hoisted(() => ({ post: vi.fn(), push: vi.fn() })); +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: state.push }) })); +vi.mock("@/components/filter-bar", () => ({ FilterBar: () => null })); +vi.mock("@/components/manual-trade-entry", () => ({ ManualTradeEntry: () => null })); +vi.mock("@/components/import-reconciliation", () => ({ ImportReconciliation: () => null })); +vi.mock("@/components/timezone-picker", () => ({ TimeZonePicker: () => null })); +vi.mock("@/components/account-picker", () => ({ + AccountPicker: ({ onChange }: { onChange: (id: string) => void }) => + createElement("button", { onClick: () => onChange("test-account") }, "Choose test account"), +})); +vi.mock("@/lib/use-api", () => ({ + postJson: (...args: unknown[]) => state.post(...args), + useApi: (url: string) => ({ + data: + url === "/api/settings" + ? { + timeZone: "UTC", + importTimeZone: "UTC", + aiProvider: "openai", + aiModel: "gpt-4.1-mini", + aiConfigured: true, + aiConnections: { + openai: { configured: true, model: "gpt-4.1-mini", source: "saved" }, + anthropic: { configured: false, model: "claude-opus-5", source: null }, + }, + } + : { formats: [] }, + }), +})); +const { default: ImportPage } = await import("../src/app/import/page"); +let container: HTMLDivElement; +let root: Root; +beforeEach(() => { + Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + state.post.mockReset(); + vi.spyOn(window, "alert").mockImplementation(() => {}); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + vi.restoreAllMocks(); +}); +const render = () => + act(async () => root.render(createElement(TooltipProvider, null, createElement(ImportPage)))); +const button = (name: string) => + Array.from(container.querySelectorAll("button")).find( + (b) => b.textContent === name, + )!; +const workbookBytes = new Uint8Array([0x50, 0x4b, 0x03, 0x04, 0xff, 0x00, 0x7f]); +const upload = (size = workbookBytes.length) => + act(async () => { + const input = container.querySelector('input[type="file"]')!; + Object.defineProperty(input, "files", { + configurable: true, + value: [{ name: "account.xlsx", size, arrayBuffer: async () => workbookBytes.buffer }], + }); + input.dispatchEvent(new Event("change", { bubbles: true })); + }); + +it("sends an XTB workbook to the server as base64 for both preview and commit", async () => { + await render(); + state.post.mockResolvedValueOnce({ + detected: "xtb", + timeZone: "UTC", + needsMapping: false, + executions: [], + warnings: [], + totals: { executions: 2, symbols: 1, skippedRows: 0, from: null, to: null }, + }); + await upload(); + const content = Buffer.from(workbookBytes).toString("base64"); + expect(state.post.mock.calls[0]![1]).toMatchObject({ + mode: "preview", + encoding: "xlsx", + content, + fileName: "account.xlsx", + }); + await act(async () => button("Choose test account").click()); + state.post.mockResolvedValueOnce({ inserted: 2, duplicates: 0 }); + await act(async () => button("Import").click()); + expect(state.post.mock.calls[1]![1]).toMatchObject({ + mode: "commit", + encoding: "xlsx", + content, + accountId: "test-account", + }); +}); + +it("keeps workbooks away from AI parsing and oversized uploads off the network", async () => { + await render(); + await act(async () => + (container.querySelector("#ai-import-enabled") as HTMLButtonElement).click(), + ); + await upload(); + expect(container.textContent).toContain("Turn off AI parsing for an XTB Excel report"); + await act(async () => + (container.querySelector("#ai-import-enabled") as HTMLButtonElement).click(), + ); + await upload(5 * 1024 * 1024 + 1); + expect(container.textContent).toContain("Excel uploads must be a .xlsx file of 5 MB or less."); + expect(state.post).not.toHaveBeenCalled(); +}); diff --git a/apps/web/tests/xtb-import.test.ts b/apps/web/tests/xtb-import.test.ts new file mode 100644 index 00000000..3e6ca214 --- /dev/null +++ b/apps/web/tests/xtb-import.test.ts @@ -0,0 +1,174 @@ +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + xtbWorkbook, + type ClosedLot, + type OpenLot, +} from "../../../packages/importers/tests/xtb-fixture"; + +const scratch = mkdtempSync(join(tmpdir(), "journal-xtb-")); +vi.stubEnv("JOURNAL_DATA_DIR", scratch); +vi.stubEnv("JOURNAL_PASSWORD", ""); +const { db, accounts, executions, trades } = await import("../src/db"); +const { POST } = await import("../src/app/api/import/route"); + +const post = (body: object) => + POST(new Request("http://localhost/api/import", { method: "POST", body: JSON.stringify(body) })); +const base64 = (bytes: Uint8Array) => Buffer.from(bytes).toString("base64"); +const upload = (mode: "preview" | "commit", bytes: Uint8Array, extra: object = {}) => + post({ + mode, + accountId: "xtb", + content: base64(bytes), + encoding: "xlsx", + fileName: "account.xlsx", + timeZone: "Europe/Bucharest", + ...extra, + }); +const state = () => ({ + executions: db.select().from(executions).all().length, + trades: db + .select() + .from(trades) + .all() + .map((trade) => [trade.status, trade.openQuantity, trade.netPnl]) + .sort(), +}); + +// Synthetic account and positions in the layout of real xStation exports. +const ACCOUNT = 10000001; +const lot = (positionId: number, volume: number, openPrice: number, openTime: string): OpenLot => ({ + instrument: "Synthetic ETF", + ticker: "SYN.DE", + category: "ETF", + type: "BUY", + volume, + openPrice, + openTime: new Date(openTime), + positionId, +}); +const lots = [ + lot(3000000002, 4.5, 750.25, "2026-10-07T17:57:32.940Z"), + lot(3000000003, 0.3, 731.1, "2026-09-30T15:07:43.479Z"), +]; +const close = (open: OpenLot, changes: Partial = {}): ClosedLot => ({ + ...open, + closePrice: open.openPrice + 10, + closeTime: new Date("2026-10-12T09:00:00.000Z"), + net: open.volume * 10, + gross: open.volume * 10, + ...changes, +}); +const holdings = xtbWorkbook(ACCOUNT, { open: lots }); +const later = (closed: ClosedLot) => + xtbWorkbook(ACCOUNT, { closed: [closed], open: lots.slice(1) }); + +beforeEach(() => { + db.delete(trades).run(); + db.delete(executions).run(); + db.delete(accounts).run(); + db.insert(accounts) + .values({ id: "xtb", name: "XTB", kind: "import", createdAt: "2026-01-01" }) + .run(); +}); +afterAll(() => { + db.$client.close(); + vi.unstubAllEnvs(); + rmSync(scratch, { recursive: true, force: true }); +}); + +describe("XTB .xlsx upload (#42)", () => { + it("previews and imports a workbook decoded on the server, once", async () => { + const preview = await (await upload("preview", holdings)).json(); + expect(preview).toMatchObject({ + detected: "xtb", + totals: { executions: 2, skippedRows: 0 }, + }); + expect(preview.errors).toBeUndefined(); + expect(await (await upload("commit", holdings)).json()).toMatchObject({ inserted: 2 }); + expect(await (await upload("commit", holdings)).json()).toMatchObject({ + inserted: 0, + duplicates: 2, + }); + expect(state()).toEqual({ + executions: 2, + trades: [ + ["open", 0.3, 0], + ["open", 4.5, 0], + ], + }); + }); + + it("closes a holding when a later workbook reports the close", async () => { + await upload("commit", holdings); + const result = await (await upload("commit", later(close(lots[0]!)))).json(); + expect(result).toMatchObject({ inserted: 1, duplicates: 2 }); + expect(state()).toEqual({ + executions: 3, + trades: [ + ["open", 0.3, 0], + ["win", 0, 45], + ], + }); + }); + + it("treats a rounding difference in the later close as the same lot", async () => { + await upload("commit", holdings); + const drifted = close(lots[0]!, { + openTime: new Date("2026-10-07T17:57:33.000Z"), + openPrice: 750.2500004, + }); + const preview = await (await upload("preview", later(drifted))).json(); + expect(preview.warnings.join(" ")).toMatch(/1 open holding\(s\) imported earlier are closed/); + expect(await (await upload("commit", later(drifted))).json()).toMatchObject({ + inserted: 1, + duplicates: 2, + }); + expect(state().trades).toEqual([ + ["open", 0.3, 0], + ["win", 0, 45], + ]); + }); + + it("refuses a later close that changes the lot instead of booking a phantom position", async () => { + await upload("commit", holdings); + const before = state(); + const partial = later(close(lots[0]!, { volume: 2, net: 20, gross: 20 })); + const preview = await (await upload("preview", partial)).json(); + expect(preview.errors.join(" ")).toMatch(/XTB position 3000000002 was imported earlier/); + const response = await upload("commit", partial); + expect(response.status).toBe(400); + expect((await response.json()).error).toMatch(/partial close.*Nothing was saved/); + expect(state()).toEqual(before); + }); + + it("refuses AI parsing, column mapping and oversized or non-workbook uploads", async () => { + const statuses = await Promise.all([ + upload("preview", holdings, { ai: { provider: "openai", model: "test" } }), + upload("preview", holdings, { mapping: { symbol: "Ticker" } }), + post({ mode: "preview", content: "not base64!", encoding: "xlsx" }), + post({ + mode: "preview", + content: base64(new TextEncoder().encode("Closed Positions")), + encoding: "xlsx", + }), + post({ + mode: "preview", + content: base64(new Uint8Array(5 * 1024 * 1024 + 1)), + encoding: "xlsx", + }), + ]); + const bodies = await Promise.all(statuses.map((response) => response.json())); + expect(statuses.map((response) => response.status)).toEqual([400, 400, 400, 400, 400]); + expect(bodies.map((body) => body.error)).toEqual([ + expect.stringMatching(/Turn off AI parsing/), + expect.stringMatching(/Turn off AI parsing/), + expect.stringMatching(/not an \.xlsx workbook/), + expect.stringMatching(/not an \.xlsx workbook/), + expect.stringMatching(/5 MB or less/), + ]); + expect(state().executions).toBe(0); + }); +}); diff --git a/docs/importers.md b/docs/importers.md index 67bc1f45..da7be80e 100644 --- a/docs/importers.md +++ b/docs/importers.md @@ -38,6 +38,7 @@ Two validation tiers: | DAS Trader Pro | fills | `Symb`/`B/S` headers | ☐ | ☐ | | MetaTrader 5 (deals report) | fills | HTML/CSV deal table signature | ☐ (fixtures) | ☐ | | TradingView (strategy list) | trades → reconstructed fills | `List of trades` headers | ☐ (fixtures) | ☐ | +| XTB xStation (account `.xlsx`) | closed trades + open lots | Closed Positions header | ☐ (fixtures) | ✅ (#42) | | Generic (column mapper) | fills | user-mapped | n/a | n/a | ¹ [TradeNote community broker parsers](https://github.com/Eleven-Trading/TradeNote/blob/main/src/utils/brokers.js): @@ -81,6 +82,38 @@ Closes are checked against the selected account's saved fills and the new file t If you previously imported the same contract after changing its actions to Buy/Sell, import the complete original history into a separate journal account and compare it before retiring the old one. The importer will not rewrite the old fills or silently combine the two representations. Export one source account at a time. Configure futures multipliers for the imported symbols in Settings; action labels do not establish a multiplier. AI parsing still does not support hedged position histories. +## XTB account workbooks + +XTB xStation exports one `.xlsx` workbook per account, with Closed Positions, +Open Positions and Cash Operations sheets. Upload it on **Import → File upload** +with AI parsing off; the server reads the workbook directly. + +- Each closed position becomes an entry fill and an exit fill at the reported + prices. **Gross Profit** is kept as the trade's result, because XTB states it + in the account currency and a conversion rate can make price × volume a + different currency. When **Profit/Loss** is lower than Gross Profit, the + difference is the fee. +- Each lot on Open Positions becomes an entry fill keyed by its XTB position id, + so it shows as an open position. The symbol rollup row is skipped, and its + category applies to the lots below it. Unrealized profit is not stored. +- Cash operations (deposits, transfers, conversions and the cash legs of stock + trades) are counted and skipped; the positions already represent the trades. +- Times labelled UTC are stored as UTC; the statement timezone does not shift them. + +**Closing a holding later.** Import a later workbook for the same XTB account +into the same journal account. A closed position whose id matches a stored open +lot reuses that lot's entry, so only the exit is added and the position closes. +Rounding differences in the open time (under a second) or price are tolerated. +If XTB reports a different size or side for that id, for example after a +partial close, the import is refused before any writes, because the journal +cannot tell which part was closed. Export one workbook whose date range covers +both the opening and the close, and import it into a new journal account. + +Limits: `.xlsx` only (not legacy `.xls` or password-protected workbooks), up to +5 MB, one XTB account per workbook and per journal account. Prices stay in the +instrument currency, so for a position quoted in another currency the trade's +result is in the account currency but its entry notional is not. + ## NinjaTrader execution exports Each source account gets a saved identity inside the selected journal account. diff --git a/packages/importers/src/formats/xtb.ts b/packages/importers/src/formats/xtb.ts index 14934105..8fad89f6 100644 --- a/packages/importers/src/formats/xtb.ts +++ b/packages/importers/src/formats/xtb.ts @@ -124,6 +124,12 @@ const positionGroup = ( openedAt: string, ) => `xtb:${account ?? "account"}:${positionId || `${symbol}:${openedAt}`}`; +/** + * A row with a type other than BUY or SELL is a trade this importer does not + * understand. Rows without a type are layout: totals, symbol rollups, notes. + */ +const isUnknownTrade = (row: Row): boolean => (pick(row, ["type"]) ?? "").trim() !== ""; + const splitSections = (table: string[][]): { closed: string[][]; open: string[][] } => { const marker = table.findIndex((row) => row[0] === SECTION); if (marker === -1) return { closed: table, open: [] }; @@ -167,7 +173,7 @@ export const xtb: ImportFormat = { if (row[Object.keys(row)[0] ?? ""] === NOTE || Object.values(row).includes(NOTE)) continue; const parsed = closedRow(row, account, accounts); if (parsed === "skip") { - if (Object.values(row).some((value) => value !== "")) skippedRows++; + if (isUnknownTrade(row)) skippedRows++; continue; } if (parsed === "invalid") { @@ -183,10 +189,17 @@ export const xtb: ImportFormat = { accountFrom(openHeader === -1 ? open : open.slice(0, openHeader)) ?? account; if (openAccount) accounts.add(openAccount); const openRecords = openHeader === -1 ? [] : toRecords(open.slice(openHeader)); + // Lot rows leave Category empty; the ticker's rollup row above them carries it. + const categories = new Map(); for (const row of openRecords) { - const holding = openRow(row, openAccount, accounts); + const ticker = pick(row, ["ticker"])?.trim().toUpperCase(); + const category = pick(row, ["category"])?.trim(); + if (ticker && category && !categories.has(ticker)) categories.set(ticker, category); + } + for (const row of openRecords) { + const holding = openRow(row, openAccount, accounts, categories); if (holding === "skip") { - if (Object.values(row).some((value) => value !== "")) skippedRows++; + if (isUnknownTrade(row)) skippedRows++; continue; } if (holding === "invalid") { @@ -288,6 +301,7 @@ const openRow = ( row: Row, account: string | undefined, accounts: Set, + categories: Map, ): ImportedExecution | "skip" | "invalid" => { const type = (pick(row, ["type"]) ?? "").trim().toLowerCase(); if (type !== "buy" && type !== "sell") return "skip"; @@ -315,7 +329,7 @@ const openRow = ( price, fee: Number.isFinite(commission) ? Math.abs(commission) : 0, executedAt: openedAt, - assetClass: assetClassOf(pick(row, ["category"])), + assetClass: assetClassOf(pick(row, ["category"])?.trim() || categories.get(symbol)), importMetadata: { id: "entry", group: positionGroup(rowAccount, positionId, symbol, openedAt), diff --git a/packages/importers/src/types.ts b/packages/importers/src/types.ts index 938b5d1a..ab29ac7d 100644 --- a/packages/importers/src/types.ts +++ b/packages/importers/src/types.ts @@ -74,7 +74,7 @@ export interface ImportFormat { } /** Turn a trade-level row into its two synthetic executions (fees on the exit). */ -export const tradeToExecutions = (trade: ImportedTrade): ImportedExecution[] => [ +export const tradeToExecutions = (trade: ImportedTrade): [ImportedExecution, ImportedExecution] => [ { symbol: trade.symbol, side: trade.direction === "long" ? "buy" : "sell", diff --git a/packages/importers/src/xlsx.ts b/packages/importers/src/xlsx.ts index 8b89df47..ee06a886 100644 --- a/packages/importers/src/xlsx.ts +++ b/packages/importers/src/xlsx.ts @@ -58,67 +58,116 @@ const columnIndex = (cellRef: string): number => { const rowNumber = (cellRef: string): number => Number(/\d+$/.exec(cellRef)?.[0] ?? 0); +/** Account statements are small; a workbook far beyond these limits is not one. */ +const MAX_ENTRIES = 1_000; +const MAX_ENTRY_BYTES = 32 * 1024 * 1024; +const MAX_TOTAL_BYTES = 64 * 1024 * 1024; + +const NOT_XLSX = "This file is not an .xlsx workbook."; +function fail(message = NOT_XLSX): never { + throw new Error(message); +} + interface ZipEntry { name: string; method: number; compressedSize: number; + uncompressedSize: number; dataOffset: number; } const findEocd = (bytes: Uint8Array): number => { const start = Math.max(0, bytes.length - 22 - 65_535); for (let offset = bytes.length - 22; offset >= start; offset--) { - if ( - bytes[offset] === 0x50 && - bytes[offset + 1] === 0x4b && - bytes[offset + 2] === 0x05 && - bytes[offset + 3] === 0x06 - ) - return offset; + if (u32(bytes, offset) === 0x06054b50) return offset; } - throw new Error("This file is not an .xlsx workbook."); + return fail(); }; +/** Central directory entries, with every offset and size checked against the file. */ const zipEntries = (bytes: Uint8Array): ZipEntry[] => { const eocd = findEocd(bytes); + if (u16(bytes, eocd + 4) !== 0 || u16(bytes, eocd + 6) !== 0) + fail("Multi-part .xlsx archives are not supported."); const count = u16(bytes, eocd + 10); let cursor = u32(bytes, eocd + 16); + if (count === 0xffff || cursor === 0xffffffff) + fail("ZIP64 workbooks are not supported. Save the file again in Excel."); + if (count > MAX_ENTRIES) fail("This workbook is too large to be an account statement."); const entries: ZipEntry[] = []; for (let index = 0; index < count; index++) { - if (u32(bytes, cursor) !== 0x02014b50) throw new Error("This file is not an .xlsx workbook."); + if (cursor + 46 > bytes.length || u32(bytes, cursor) !== 0x02014b50) fail(); + const flags = u16(bytes, cursor + 8); const method = u16(bytes, cursor + 10); const compressedSize = u32(bytes, cursor + 20); + const uncompressedSize = u32(bytes, cursor + 24); const nameLength = u16(bytes, cursor + 28); const extraLength = u16(bytes, cursor + 30); const commentLength = u16(bytes, cursor + 32); const localOffset = u32(bytes, cursor + 42); + if (flags & 0x1) + fail( + "Password-protected workbooks are not supported. Export the report again without a password.", + ); + if ([compressedSize, uncompressedSize, localOffset].includes(0xffffffff)) + fail("ZIP64 workbooks are not supported. Save the file again in Excel."); + if (uncompressedSize > MAX_ENTRY_BYTES) + fail("This workbook is too large to be an account statement."); + if (cursor + 46 + nameLength > bytes.length) fail(); + if (localOffset + 30 > bytes.length || u32(bytes, localOffset) !== 0x04034b50) fail(); const name = new TextDecoder().decode(bytes.subarray(cursor + 46, cursor + 46 + nameLength)); - const localNameLength = u16(bytes, localOffset + 26); - const localExtraLength = u16(bytes, localOffset + 28); - entries.push({ - name, - method, - compressedSize, - dataOffset: localOffset + 30 + localNameLength + localExtraLength, - }); + const dataOffset = + localOffset + 30 + u16(bytes, localOffset + 26) + u16(bytes, localOffset + 28); + if (dataOffset + compressedSize > bytes.length) fail(); + entries.push({ name, method, compressedSize, uncompressedSize, dataOffset }); cursor += 46 + nameLength + extraLength + commentLength; } return entries; }; -const inflateRaw = async (data: Uint8Array): Promise => { +/** Inflate, stopping as soon as the output passes the size the archive declared. */ +const inflateRaw = async (data: Uint8Array, limit: number): Promise => { if (data.length === 0) return new Uint8Array(); - const stream = new Blob([data.slice()]) + const reader = new Blob([data.slice()]) .stream() - .pipeThrough(new DecompressionStream("deflate-raw")); - return new Uint8Array(await new Response(stream).arrayBuffer()); + .pipeThrough(new DecompressionStream("deflate-raw")) + .getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.length; + if (size > limit) { + await reader.cancel(); + fail("This workbook is damaged: a part is larger than the archive declares."); + } + chunks.push(value); + } + const output = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + output.set(chunk, offset); + offset += chunk.length; + } + return output; }; const readEntry = async (bytes: Uint8Array, entry: ZipEntry): Promise => { const compressed = bytes.subarray(entry.dataOffset, entry.dataOffset + entry.compressedSize); - if (entry.method === 0) return new TextDecoder().decode(compressed); - if (entry.method === 8) return new TextDecoder().decode(await inflateRaw(compressed)); - throw new Error(`Unsupported Excel compression (method ${entry.method}).`); + if (entry.method === 0) { + if (compressed.length > entry.uncompressedSize) fail(); + return new TextDecoder().decode(compressed); + } + if (entry.method !== 8) fail(`Unsupported Excel compression (method ${entry.method}).`); + let inflated: Uint8Array; + try { + inflated = await inflateRaw(compressed, entry.uncompressedSize); + } catch (error) { + if (error instanceof Error && error.message.startsWith("This workbook")) throw error; + return fail("This .xlsx workbook is damaged and could not be read."); + } + return new TextDecoder().decode(inflated); }; const sharedStrings = (xml: string): string[] => @@ -196,17 +245,25 @@ const sheetRows = ( /** Read every worksheet. Dates become UTC ISO strings; other numbers stay plain decimals. */ export const readXlsx = async (data: ArrayBuffer | Uint8Array): Promise => { const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); - if (bytes.length < 4 || bytes[0] !== 0x50 || bytes[1] !== 0x4b) - throw new Error("This file is not an .xlsx workbook."); + // Legacy .xls files and password-protected .xlsx files are OLE compound documents. + if (u32(bytes, 0) === 0xe011cfd0) + fail( + "This is a legacy .xls or password-protected workbook. Export the report again as .xlsx without a password.", + ); + if (bytes.length < 22 || u32(bytes, 0) !== 0x04034b50) fail(); const entries = zipEntries(bytes); const byName = new Map(entries.map((entry) => [entry.name.replace(/\\/g, "/"), entry])); + let expanded = 0; const read = async (name: string): Promise => { const entry = byName.get(name) ?? byName.get(`xl/${name.replace(/^xl\//, "")}`); - return entry ? readEntry(bytes, entry) : null; + if (!entry) return null; + expanded += entry.uncompressedSize; + if (expanded > MAX_TOTAL_BYTES) fail("This workbook is too large to be an account statement."); + return readEntry(bytes, entry); }; const workbook = await read("xl/workbook.xml"); const rels = await read("xl/_rels/workbook.xml.rels"); - if (!workbook || !rels) throw new Error("This file is not an .xlsx workbook."); + if (!workbook || !rels) fail(); const strings = sharedStrings((await read("xl/sharedStrings.xml")) ?? ""); const dates = dateStyleIndexes((await read("xl/styles.xml")) ?? ""); const epochMs = /date1904="(?:1|true)"/.test(workbook) ? Date.UTC(1904, 0, 1) : EXCEL_EPOCH_MS; @@ -227,6 +284,6 @@ export const readXlsx = async (data: ArrayBuffer | Uint8Array): Promise + text.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); +const column = (index: number): string => + index < 26 + ? String.fromCharCode(65 + index) + : column(Math.floor(index / 26) - 1) + column(index % 26); + +const sheetXml = (rows: Cell[][], strings: Map): string => { + const shared = (text: string) => { + if (!strings.has(text)) strings.set(text, strings.size); + return strings.get(text)!; + }; + const body = rows + .map((cells, r) => { + const xml = cells + .map((cell, c) => { + const ref = `${column(c)}${r + 1}`; + if (cell === null) return ""; + if (cell instanceof Date) + return `${(cell.getTime() - EXCEL_EPOCH) / 86_400_000}`; + if (typeof cell === "number") return `${cell}`; + return `${shared(cell)}`; + }) + .join(""); + return `${xml}`; + }) + .join(""); + return ` +${body}`; +}; + +/** A minimal but valid .xlsx package; `stored` skips compression. */ +export const workbook = (sheets: Sheet[], options: { stored?: boolean } = {}): Uint8Array => { + const strings = new Map(); + const sheetParts = sheets.map((sheet, index) => ({ + name: `xl/worksheets/sheet${index + 1}.xml`, + xml: sheetXml(sheet.rows, strings), + })); + const parts: { name: string; xml: string }[] = [ + { + name: "[Content_Types].xml", + xml: `${sheetParts.map((part) => ``).join("")}`, + }, + { + name: "_rels/.rels", + xml: ``, + }, + { + name: "xl/workbook.xml", + xml: `${sheets.map((sheet, index) => ``).join("")}`, + }, + { + name: "xl/_rels/workbook.xml.rels", + xml: `${sheets.map((_, index) => ``).join("")}`, + }, + ...sheetParts, + { + name: "xl/styles.xml", + xml: ``, + }, + ]; + parts.push({ + name: "xl/sharedStrings.xml", + xml: `${[...strings.keys()].map((text) => `${escape(text)}`).join("")}`, + }); + return zip( + parts.map(({ name, xml }) => ({ name, data: Buffer.from(xml) })), + options.stored, + ); +}; + +const zip = (files: { name: string; data: Buffer }[], stored = false): Uint8Array => { + const locals: Buffer[] = []; + const centrals: Buffer[] = []; + let offset = 0; + for (const file of files) { + const name = Buffer.from(file.name); + const body = stored ? file.data : deflateRawSync(file.data); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(stored ? 0 : 8, 8); + local.writeUInt32LE(crc32(file.data), 14); + local.writeUInt32LE(body.length, 18); + local.writeUInt32LE(file.data.length, 22); + local.writeUInt16LE(name.length, 26); + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(20, 4); + central.writeUInt16LE(20, 6); + central.writeUInt16LE(stored ? 0 : 8, 10); + central.writeUInt32LE(crc32(file.data), 16); + central.writeUInt32LE(body.length, 20); + central.writeUInt32LE(file.data.length, 24); + central.writeUInt16LE(name.length, 28); + central.writeUInt32LE(offset, 42); + locals.push(local, name, body); + centrals.push(central, name); + offset += local.length + name.length + body.length; + } + const directory = Buffer.concat(centrals); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(files.length, 8); + end.writeUInt16LE(files.length, 10); + end.writeUInt32LE(directory.length, 12); + end.writeUInt32LE(offset, 16); + return new Uint8Array(Buffer.concat([...locals, directory, end])); +}; + +const CLOSED_HEADER = [ + "Instrument", + "Ticker", + "Category", + "Type", + "Volume", + "Open Price", + "Open Time (UTC)", + "Close Price", + "Close Time (UTC)", + "Product", + "Profit/Loss", + "Gross Profit", + "Purchase Value", + "Sale Value", + "Stop Loss", + "Take Profit", + "Commission", + "Margin", + "Swap", + "Rollover", + "Open Conversion Rate", + "Close Conversion Rate", + "Close Origin", + "Position ID", + "Comment", +]; +const CASH_HEADER = [ + "Type", + "Instrument", + "Ticker", + "Category", + "Time", + "Amount", + "ID", + "Comment", + "Product", + "Position ID", +]; +const OPEN_HEADER = [ + "Product", + "Instrument/Position", + "Ticker", + "Category", + "Type", + "Volume", + "Value", + "Current price", + "Open price", + "Open time (UTC)", + "Stop Loss", + "Take Profit", + "Net Profit %", + "Net Profit", + "Gross Profit", + "Margin", + "Open Commission", + "Swap", + "Rollover", +]; + +export interface ClosedLot { + instrument: string; + ticker: string; + category: string; + type: "BUY" | "SELL"; + volume: number; + openPrice: number; + openTime: Date; + closePrice: number; + closeTime: Date; + net: number; + gross: number; + commission?: number; + openRate?: number; + closeRate?: number; + positionId: number; +} +export interface OpenLot { + instrument: string; + ticker: string; + category: string; + type: "BUY" | "SELL"; + volume: number; + openPrice: number; + openTime: Date; + positionId: number; +} +export interface CashOperation { + type: string; + time: Date; + amount: number; + id: number; + comment: string; + ticker?: string; + positionId?: number; +} + +/** One XTB account workbook: Closed Positions, Cash Operations, Open Positions. */ +export const xtbWorkbook = ( + account: number, + data: { closed?: ClosedLot[]; open?: OpenLot[]; cash?: CashOperation[] }, + options: { stored?: boolean } = {}, +): Uint8Array => { + const from = new Date("2026-09-01T21:00:00.000Z"); + const to = new Date("2026-10-08T08:00:00.000Z"); + const closed = data.closed ?? []; + const open = data.open ?? []; + const cash = data.cash ?? []; + const preamble = (title: string): Cell[][] => [ + ["Account number", account], + [title], + ["Date from (UTC)", from], + ["Date to (UTC)", to], + ]; + const sum = (values: number[]) => values.reduce((total, value) => total + value, 0); + const closedRows: Cell[][] = closed.map((lot) => [ + lot.instrument, + lot.ticker, + lot.category, + lot.type, + lot.volume, + lot.openPrice, + lot.openTime, + lot.closePrice, + lot.closeTime, + "Investment Plans", + lot.net, + lot.gross, + lot.volume * lot.openPrice * (lot.openRate ?? 1), + lot.volume * lot.closePrice * (lot.closeRate ?? 1), + "", + "", + lot.commission ?? 0, + "", + "", + "", + lot.openRate ?? 1, + lot.closeRate ?? 1, + "xStation5", + lot.positionId, + "", + ]); + const lotValue = (lot: OpenLot) => Number((lot.volume * lot.openPrice).toFixed(2)); + const openRows: Cell[][] = open.length + ? [ + [ + "Investment Plan", + open[0]!.instrument, + open[0]!.ticker, + open[0]!.category, + "", + sum(open.map((lot) => lot.volume)), + sum(open.map(lotValue)), + "", + open[0]!.openPrice, + "", + "", + "", + 0, + 0, + 0, + ], + ...open.map((lot): Cell[] => [ + "Investment Plan", + lot.positionId, + lot.ticker, + "", + lot.type, + lot.volume, + lotValue(lot), + lot.openPrice, + lot.openPrice, + lot.openTime, + "", + "", + 0, + 0, + 0, + ]), + ] + : []; + return workbook( + [ + { + name: "Closed Positions", + rows: [ + ...preamble("Closed Positions"), + CLOSED_HEADER, + ...closedRows, + [ + "Profit/loss", + ...Array(9).fill(""), + sum(closed.map((lot) => lot.net)), + sum(closed.map((lot) => lot.gross)), + ], + ], + }, + { + name: "Cash Operations", + rows: [ + ...preamble("Cash Operations"), + CASH_HEADER, + ...cash.map((operation): Cell[] => [ + operation.type, + operation.ticker ? "Synthetic ETF" : "", + operation.ticker ?? "", + operation.ticker ? "ETF" : "", + operation.time, + operation.amount, + operation.id, + operation.comment, + "My Trades", + ...(operation.positionId ? [operation.positionId] : []), + ]), + ["Total", "", "", "", "", sum(cash.map((operation) => operation.amount))], + ], + }, + { + name: "Open Positions", + rows: [ + ["Account number", account], + ["Open Positions"], + ["Data as of report generated", to], + ["Product", "Metric", "Amount", "Currency"], + ["Investment Plans", "Open position value", sum(open.map(lotValue)), "EUR"], + ["Investment Plans", "Open position profit", 0, "EUR"], + ["Note", "Summary values and open positions are shown as of the report generation time"], + OPEN_HEADER, + ...openRows, + ], + }, + ], + options, + ); +}; diff --git a/packages/importers/tests/xtb.test.ts b/packages/importers/tests/xtb.test.ts new file mode 100644 index 00000000..b064330a --- /dev/null +++ b/packages/importers/tests/xtb.test.ts @@ -0,0 +1,373 @@ +import { describe, expect, it } from "vitest"; +import { buildRoundTrips } from "@luxalgo/journal-core"; +import { detectFormat, parseAuto } from "../src/detect"; +import { xtbWorkbookToStatement } from "../src/formats/xtb"; +import { readXlsx } from "../src/xlsx"; +import { workbook, xtbWorkbook, type OpenLot } from "./xtb-fixture"; + +// Fixtures are synthetic workbooks in the layout of real xStation exports. +const ACCOUNT = 10000001; +const at = (iso: string) => new Date(iso); + +const closedEtf = xtbWorkbook(ACCOUNT, { + closed: [ + { + instrument: "Synthetic ETF", + ticker: "SYN.DE", + category: "ETF", + type: "BUY", + volume: 0.25, + openPrice: 700, + openTime: at("2026-09-30T13:38:21.147Z"), + closePrice: 699.5, + closeTime: at("2026-09-30T13:47:46.059Z"), + // Account-currency result: 0.25 × −0.50 EUR × 5.0 RON/EUR. + net: -0.63, + gross: -0.63, + openRate: 5, + closeRate: 5, + positionId: 3000000001, + }, + ], + cash: [ + { + type: "Deposit", + time: at("2026-09-30T13:31:07.967Z"), + amount: 1000, + id: 1, + comment: "Deposit", + }, + { + type: "Stock purchase", + time: at("2026-09-30T13:38:21.157Z"), + amount: -875, + id: 2, + comment: "OPEN BUY SYN.DE 0.25 @ 700.00", + ticker: "SYN.DE", + positionId: 3000000001, + }, + { + type: "Stock sell", + time: at("2026-09-30T13:47:46.070Z"), + amount: 874.37, + id: 3, + comment: "CLOSE BUY SYN.DE 0.25 @ 699.50", + ticker: "SYN.DE", + positionId: 3000000001, + }, + ], +}); + +const lots: OpenLot[] = [ + { + instrument: "Synthetic ETF", + ticker: "SYN.DE", + category: "ETF", + type: "BUY", + volume: 4.5, + openPrice: 750.25, + openTime: at("2026-10-07T17:57:32.940Z"), + positionId: 3000000002, + }, + { + instrument: "Synthetic ETF", + ticker: "SYN.DE", + category: "ETF", + type: "BUY", + volume: 0.3, + openPrice: 731.1, + openTime: at("2026-09-30T15:07:43.479Z"), + positionId: 3000000003, + }, + { + instrument: "Synthetic ETF", + ticker: "SYN.DE", + category: "ETF", + type: "BUY", + volume: 0.15, + openPrice: 738.6, + openTime: at("2026-10-02T17:37:36.856Z"), + positionId: 3000000004, + }, +]; +const purchases = lots.map((lot, index) => ({ + type: "Stock purchase", + time: lot.openTime, + amount: -lot.volume * lot.openPrice, + id: 10 + index, + comment: `OPEN BUY SYN.DE ${lot.volume} @ ${lot.openPrice}`, + ticker: "SYN.DE", + positionId: lot.positionId, +})); +const holdings = xtbWorkbook(ACCOUNT, { + open: lots, + cash: [ + { + type: "Deposit", + time: at("2026-09-30T13:50:50.915Z"), + amount: 5000, + id: 9, + comment: "Deposit", + }, + ...purchases, + ], +}); +// A later export of the same account: the first lot has been closed. +const laterClose = xtbWorkbook(ACCOUNT, { + closed: [ + { + instrument: "Synthetic ETF", + ticker: "SYN.DE", + category: "ETF", + type: "BUY", + volume: 4.5, + openPrice: 750.25, + openTime: at("2026-10-07T17:57:32.940Z"), + closePrice: 760.25, + closeTime: at("2026-10-12T09:00:00.000Z"), + net: 45, + gross: 45, + positionId: 3000000002, + }, + ], + open: lots.slice(1), +}); + +const asExecutions = (content: string) => + parseAuto(content)!.executions.map((execution, index) => ({ + ...execution, + id: `e${index}`, + accountId: "a", + source: "import" as const, + })); +const tripsOf = (content: string) => { + const parsed = parseAuto(content)!; + return { parsed, trips: buildRoundTrips(asExecutions(content)) }; +}; + +describe("XTB account workbooks", () => { + it("keeps a closed position and its account-currency result", async () => { + const statement = await xtbWorkbookToStatement(closedEtf); + expect(detectFormat(statement)?.id).toBe("xtb"); + const { parsed, trips } = tripsOf(statement); + expect(parsed.format).toBe("xtb"); + expect(parsed.errors).toBeUndefined(); + // Totals, rollups and the reducer's note are layout, not skipped trades. + expect(parsed.skippedRows).toBe(0); + expect(parsed.executions).toEqual([ + expect.objectContaining({ + symbol: "SYN.DE", + side: "buy", + quantity: 0.25, + price: 700, + fee: 0, + executedAt: "2026-09-30T13:38:21.147Z", + assetClass: "equity", + importMetadata: expect.objectContaining({ id: "entry", group: "xtb:10000001:3000000001" }), + }), + expect.objectContaining({ + symbol: "SYN.DE", + side: "sell", + quantity: 0.25, + price: 699.5, + fee: 0, + executedAt: "2026-09-30T13:47:46.059Z", + importMetadata: expect.objectContaining({ + id: "exit", + reportedGrossPnl: -0.63, + preserveFee: true, + group: "xtb:10000001:3000000001", + }), + }), + ]); + expect(trips).toHaveLength(1); + expect(trips[0]).toMatchObject({ + direction: "long", + status: "loss", + grossPnl: -0.63, + fees: 0, + netPnl: -0.63, + }); + expect(parsed.warnings.join(" ")).toMatch( + /Imported 0 open holding\(s\) as entry fills\. Skipped 3 cash operation/, + ); + }); + + it("imports open lots as holdings keyed by XTB position id, skipping the rollup row", async () => { + const { parsed, trips } = tripsOf(await xtbWorkbookToStatement(holdings)); + expect(parsed.errors).toBeUndefined(); + expect(parsed.skippedRows).toBe(0); + expect( + parsed.executions.map((execution) => [ + execution.symbol, + execution.side, + execution.quantity, + execution.price, + execution.executedAt, + execution.importMetadata?.group, + ]), + ).toEqual( + lots.map((lot) => [ + "SYN.DE", + "buy", + lot.volume, + lot.openPrice, + lot.openTime.toISOString(), + `xtb:10000001:${lot.positionId}`, + ]), + ); + // Lot rows leave Category blank; the rollup row's category still applies. + expect(parsed.executions.map((execution) => execution.assetClass)).toEqual([ + "equity", + "equity", + "equity", + ]); + expect(trips.map((trip) => trip.status)).toEqual(["open", "open", "open"]); + expect(parsed.warnings.join(" ")).toMatch(/Imported 3 open holding\(s\).*Skipped 4 cash/); + }); + + it("a later workbook's closed row repeats the holding's entry exactly and adds the exit", async () => { + const [holding] = parseAuto(await xtbWorkbookToStatement(holdings))!.executions; + const later = parseAuto(await xtbWorkbookToStatement(laterClose))!.executions; + expect(later.map((fill) => [fill.importMetadata?.id, fill.importMetadata?.group])).toEqual([ + ["entry", "xtb:10000001:3000000002"], + ["exit", "xtb:10000001:3000000002"], + ["entry", "xtb:10000001:3000000003"], + ["entry", "xtb:10000001:3000000004"], + ]); + // Same fields, so the journal's content hash deduplicates the entry. + expect(later[0]).toEqual(holding); + const trips = buildRoundTrips( + [holding!, later[1]!].map((execution, index) => ({ + ...execution, + id: `e${index}`, + accountId: "a", + source: "import" as const, + })), + ); + expect(trips).toEqual([ + expect.objectContaining({ status: "win", netPnl: 45, openQuantity: 0 }), + ]); + }); + + it("keeps overlapping long and short positions apart and preserves a fee", () => { + const csv = `Closed Positions +Instrument,Ticker,Category,Type,Volume,Open Price,Open Time (UTC),Close Price,Close Time (UTC),Profit/Loss,Gross Profit,Position ID +EURUSD,EURUSD,Forex,BUY,1,1.1,2026-01-05 09:00:00,1.2,2026-01-05 12:00:00,90,100,1 +EURUSD,EURUSD,Forex,SELL,1,1.1,2026-01-05 10:00:00,1.0,2026-01-05 11:00:00,90,100,2`; + const { parsed, trips } = tripsOf(csv); + expect(parsed.format).toBe("xtb"); + expect(trips.map((trip) => [trip.direction, trip.netPnl, trip.fees])).toEqual([ + ["long", 90, 10], + ["short", 90, 10], + ]); + expect(parsed.executions[1]?.importMetadata?.reportedGrossPnl).toBe(100); + expect(parsed.executions[1]?.fee).toBe(10); + }); + + it("counts a trade type it does not understand as skipped", () => { + const csv = `Closed Positions +Instrument,Ticker,Category,Type,Volume,Open Price,Open Time (UTC),Close Price,Close Time (UTC),Profit/Loss,Gross Profit,Position ID +AAA,AAA,ETF,BUY,1,10,2026-01-01 00:00:00,11,2026-01-02 00:00:00,1,1,10 +AAA,AAA,ETF,EXPIRY,1,10,2026-01-01 00:00:00,0,2026-01-03 00:00:00,-10,-10,11 +Profit/loss,,,,,,,,,-9,-9`; + const parsed = parseAuto(csv)!; + expect(parsed.executions).toHaveLength(2); + expect(parsed.skippedRows).toBe(1); + }); + + it("rejects a statement that mixes two XTB accounts", () => { + const csv = `Account number,1 +Closed Positions +Instrument,Ticker,Category,Type,Volume,Open Price,Open Time (UTC),Close Price,Close Time (UTC),Profit/Loss,Gross Profit,Position ID,Account +A,AAA,ETF,BUY,1,10,2026-01-01 00:00:00,11,2026-01-02 00:00:00,1,1,10,1 +A,AAA,ETF,BUY,1,10,2026-01-01 00:00:00,11,2026-01-02 00:00:00,1,1,11,2`; + expect(parseAuto(csv)!.errors?.[0]).toMatch(/more than one XTB account/); + }); + + it("refuses a workbook that is not an XTB account statement", async () => { + const other = workbook([ + { + name: "Sheet1", + rows: [ + ["Date", "Symbol"], + ["2026-01-01", "AAA"], + ], + }, + ]); + await expect(xtbWorkbookToStatement(other)).rejects.toThrow(/no Closed Positions sheet/); + }); +}); + +/** Overwrite a central-directory field of one archive member. */ +const patchCentral = ( + bytes: Uint8Array, + member: string, + write: (view: DataView, at: number) => void, +) => { + const copy = bytes.slice(); + const view = new DataView(copy.buffer); + const text = new TextDecoder(); + for (let at = 0; at < copy.length - 46; at++) { + if (view.getUint32(at, true) !== 0x02014b50) continue; + const nameLength = view.getUint16(at + 28, true); + if (text.decode(copy.subarray(at + 46, at + 46 + nameLength)) === member) { + write(view, at); + return copy; + } + } + throw new Error(`${member} not found`); +}; + +describe(".xlsx reader limits", () => { + it("reads stored and deflated workbooks identically", async () => { + const stored = xtbWorkbook(ACCOUNT, { open: lots }, { stored: true }); + const deflated = xtbWorkbook(ACCOUNT, { open: lots }); + expect(await readXlsx(stored)).toEqual(await readXlsx(deflated)); + }); + + it("names a legacy .xls or password-protected file", async () => { + const ole = new Uint8Array([ + 0xd0, + 0xcf, + 0x11, + 0xe0, + 0xa1, + 0xb1, + 0x1a, + 0xe1, + ...new Array(64).fill(0), + ]); + await expect(readXlsx(ole)).rejects.toThrow(/legacy \.xls or password-protected/); + }); + + it("rejects truncated and non-zip input", async () => { + await expect(readXlsx(closedEtf.subarray(0, closedEtf.length - 30))).rejects.toThrow( + /not an \.xlsx workbook/, + ); + await expect(readXlsx(new TextEncoder().encode("Closed Positions,,,"))).rejects.toThrow( + /not an \.xlsx workbook/, + ); + }); + + it("stops inflating a part that grows past its declared size", async () => { + const lying = patchCentral(closedEtf, "xl/worksheets/sheet1.xml", (view, at) => + view.setUint32(at + 24, 64, true), + ); + await expect(readXlsx(lying)).rejects.toThrow(/larger than the archive declares/); + }); + + it("refuses parts declared too large for an account statement", async () => { + const huge = patchCentral(closedEtf, "xl/worksheets/sheet1.xml", (view, at) => + view.setUint32(at + 24, 512 * 1024 * 1024, true), + ); + await expect(readXlsx(huge)).rejects.toThrow(/too large to be an account statement/); + }); + + it("refuses encrypted archive members", async () => { + const locked = patchCentral(closedEtf, "xl/workbook.xml", (view, at) => + view.setUint16(at + 8, 1, true), + ); + await expect(readXlsx(locked)).rejects.toThrow(/Password-protected/); + }); +}); diff --git a/packages/importers/tsconfig.json b/packages/importers/tsconfig.json index 564a5990..8128190d 100644 --- a/packages/importers/tsconfig.json +++ b/packages/importers/tsconfig.json @@ -1,4 +1,8 @@ { "extends": "../../tsconfig.base.json", + "compilerOptions": { + // Runs in Node and the browser: the .xlsx reader uses TextDecoder and DecompressionStream. + "lib": ["ES2022", "DOM"] + }, "include": ["src"] }