Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ All notable changes to this project are documented here. The format follows [Kee
- PDF and PNG review exports; JSON export now includes folders, templates, rule assessments, routines and defaults
- Configurable breakeven tolerance, default fees, and default stop/target distances per account and symbol
- Importers: MetaTrader 5 deal reports and TradingView strategy exports
- Importers: XTB xStation `.xlsx` account workbooks, with closed positions, open holdings and later closes of those holdings (contributed by @zahei, [#42](https://github.com/LuxAlgo/trade-journal/issues/42))
- Realized R accounts for contract multipliers and scaled entries
- Optional historical market data connections (London Strategic Edge, Alpaca, OANDA, Binance, Coinbase, or your own candle CSV) with estimated MAE/MFE and candle replay on closed trades
- Prop firm tracker: evaluation and reset costs, refunds, payout requests, receipts, cash ROI, renewals, attachments, CSV import and export
Expand Down
32 changes: 16 additions & 16 deletions README.md

Large diffs are not rendered by default.

50 changes: 35 additions & 15 deletions apps/web/src/app/api/import/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
looksLikeFlexXml,
parseFlexXmlImport,
} from "@/server/ibkr-flex-xml-import";
import { reconcileXtbHoldings, xlsxStatement } from "@/server/xtb-import";
import type { AiImportOptions } from "@/lib/ai-import";
import {
createAiImportPreview,
Expand All @@ -36,7 +37,7 @@ interface ImportBody {
symbol?: string;
review?: ImportReviewOptions;
ai?: AiImportOptions;
encoding?: "text" | "pdf";
encoding?: "text" | "pdf" | "xlsx";
aiPreviewToken?: string;
aiReviewed?: boolean;
}
Expand All @@ -58,15 +59,22 @@ export const POST = handler(async (request: Request) => {
requireValue(isTimeZone(body.timeZone), "Enter a valid IANA statement timezone.");
const timeZone = body.timeZone ?? getImportTimeZone();
requireValue(
body.encoding === undefined || body.encoding === "text" || body.encoding === "pdf",
body.encoding === undefined || ["text", "pdf", "xlsx"].includes(body.encoding),
"Unsupported file encoding.",
);
requireValue(body.encoding !== "pdf" || body.ai, "Enable AI parsing to upload PDF statements.");
requireValue(
body.encoding !== "xlsx" || (!body.ai && !body.mapping),
"Turn off AI parsing for an XTB Excel report. The XTB importer reads it directly.",
);
requireValue(!body.ai || !body.mapping, "Choose AI parsing or column mapping, not both.");
// Workbooks are reduced to a statement here, so preview and commit parse the same server-side text.
const content = body.encoding === "xlsx" ? await xlsxStatement(body.content) : body.content;
const statement = {
content: body.content,
fileName: body.fileName,
encoding: body.encoding,
// Only AI parsing reads this, and a workbook with AI parsing is refused above.
encoding: body.encoding === "xlsx" ? undefined : body.encoding,
timeZone,
};
if (body.ai && body.mode === "commit")
Expand All @@ -80,16 +88,16 @@ export const POST = handler(async (request: Request) => {
? await parseStatementWithAi(statement, body.ai, request.signal)
: readAiImportPreview(statement, body.aiPreviewToken!)
: body.mapping
? parseWithMapping(body.content, body.mapping, { timeZone })
: looksLikeFlexXml(body.content)
? parseFlexXmlImport(body.content, timeZone)
: parseAuto(body.content, { timeZone, fileName: body.fileName, symbol: body.symbol });
? parseWithMapping(content, body.mapping, { timeZone })
: looksLikeFlexXml(content)
? parseFlexXmlImport(content, timeZone)
: parseAuto(content, { timeZone, fileName: body.fileName, symbol: body.symbol });

if (!parsed) {
return ok({
detected: null,
timeZone,
headers: readHeaders(body.content),
headers: readHeaders(content),
needsMapping: true,
});
}
Expand All @@ -98,7 +106,18 @@ export const POST = handler(async (request: Request) => {
parsed.format === IBKR_FLEX_XML.id && body.accountId
? flexUploadSyncProblem(body.accountId, timeZone)
: null;
const errors = [...(parsed.errors ?? []), ...(syncProblem ? [syncProblem] : [])];
const xtb =
parsed.format === "xtb" && body.accountId
? reconcileXtbHoldings(body.accountId, parsed.executions as ImportedExecution[])
: null;
const problems = [syncProblem, xtb?.problem].filter((problem): problem is string => !!problem);
const errors = [...(parsed.errors ?? []), ...problems];
const warnings = [
...(parsed.warnings ?? []),
...(xtb?.matched
? [`${xtb.matched} open holding(s) imported earlier are closed by this file.`]
: []),
];

if (body.mode === "preview") {
const symbols = [...new Set(parsed.executions.map((e) => e.symbol))];
Expand Down Expand Up @@ -126,8 +145,8 @@ export const POST = handler(async (request: Request) => {
null,
),
},
warnings: parsed.warnings,
errors: syncProblem ? errors : parsed.errors,
warnings,
errors: problems.length ? errors : parsed.errors,
needsSymbol: parsed.needsSymbol,
reconciliation:
parsed.format === "ninjatrader" && body.accountId
Expand All @@ -144,21 +163,22 @@ export const POST = handler(async (request: Request) => {
? commitNinjaTraderImport(body.accountId, parsed, body.content, timeZone, body.review)
: insertExecutions(
body.accountId,
parsed.executions as ImportedExecution[],
xtb?.executions ?? (parsed.executions as ImportedExecution[]),
"import",
undefined,
{ preserveFees: Boolean(body.ai) },
);
if (xtb?.matched) result.duplicates += xtb.matched;
// Invalid rows are skipped with a warning rather than failing the whole file.
const warnings = [
...(parsed.warnings ?? []),
const commitWarnings = [
...warnings,
...(result.skipped > 0
? [
`${result.skipped} row(s) were skipped because they could not be journaled: ${result.skippedReasons.join(" ")}`,
]
: []),
];
return ok({ detected: parsed.format, ...result, warnings });
return ok({ detected: parsed.format, ...result, warnings: commitWarnings });
});

/** The import page lists what auto-detection understands. */
Expand Down
29 changes: 22 additions & 7 deletions apps/web/src/app/import/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import {
AI_IMPORT_MAX_TEXT,
type AiImportOptions as AiOptions,
} from "@/lib/ai-import";
import { XLSX_MAX_BYTES } from "@/lib/xlsx-import";
import { Checkbox } from "@/components/ui/checkbox";

interface BrokerInfo {
Expand Down Expand Up @@ -155,7 +156,7 @@ function FileImport() {
model: AI_DEFAULT_MODELS.openai,
apiKey: "",
});
const [encoding, setEncoding] = useState<"text" | "pdf">("text");
const [encoding, setEncoding] = useState<"text" | "pdf" | "xlsx">("text");
const [aiReviewed, setAiReviewed] = useState(false);
const { data: formatData } = useApi<{ formats: { id: string; label: string }[] }>("/api/import");
const { data: settingsData, error: settingsError } = useApi<
Expand Down Expand Up @@ -193,16 +194,24 @@ function FileImport() {
setBusy(true);
try {
const pdf = /\.pdf$/i.test(file.name);
const xlsx = /\.xlsx$/i.test(file.name);
if (pdf && !aiEnabled) throw new Error("Enable AI parsing to upload a PDF statement.");
if (xlsx && aiEnabled)
throw new Error(
"Turn off AI parsing for an XTB Excel report. The XTB importer reads it directly.",
);
if (aiEnabled && file.size > AI_IMPORT_MAX_BYTES)
throw new Error("AI uploads must be 8 MB or smaller.");
if (xlsx && file.size > XLSX_MAX_BYTES)
throw new Error("Excel uploads must be a .xlsx file of 5 MB or less.");
if (aiEnabled && !/\.(csv|tsv|txt|html?|xml|pdf)$/i.test(file.name))
throw new Error(
"Choose a CSV, TSV, HTML, XML, TXT or PDF file. Export spreadsheets as CSV first.",
"Choose a CSV, TSV, HTML, XML, TXT or PDF file. XTB Excel reports use the XTB importer with AI turned off.",
);
const buffer = await file.arrayBuffer();
let text: string;
if (pdf) {
// Binary statements travel as base64; the server reads them.
if (pdf || xlsx) {
const bytes = new Uint8Array(buffer);
let binary = "";
for (let i = 0; i < bytes.length; i += 8192)
Expand All @@ -215,7 +224,8 @@ function FileImport() {
"Use a smaller export: AI text parsing supports up to 150,000 characters.",
);
}
setEncoding(pdf ? "pdf" : "text");
const fileEncoding = pdf ? "pdf" : xlsx ? "xlsx" : "text";
setEncoding(fileEncoding);
setContent(text);
// Choosing a file alone never sends it to an AI provider.
if (aiEnabled) return;
Expand All @@ -227,6 +237,7 @@ function FileImport() {
review: {},
fileName: file.name,
timeZone,
...(fileEncoding === "xlsx" ? { encoding: fileEncoding } : {}),
}),
);
} catch (cause) {
Expand Down Expand Up @@ -304,6 +315,7 @@ function FileImport() {
symbol,
// Commit with the exact parsing zone used by the reviewed preview.
timeZone: preview.timeZone,
...(encoding === "xlsx" ? { encoding } : {}),
...(preview.aiPreviewToken
? {
ai: { provider: aiOptions.provider, model: aiOptions.model },
Expand Down Expand Up @@ -395,7 +407,7 @@ function FileImport() {
{fileName ||
(aiEnabled
? "Choose a statement for AI parsing"
: "Choose a CSV / HTML / XML statement")}
: "Choose a CSV / HTML / XML statement or an XTB Excel report")}
</span>
<span className="text-xs text-muted-foreground">
{aiEnabled ? (
Expand All @@ -408,14 +420,17 @@ function FileImport() {
formatData?.formats.map((format) => format.label.split(" (")[0]) ?? [],
),
].join(", ")}{" "}
— anything else via column mapping.
— anything else via column mapping. XTB: one .xlsx per account. Closed positions
and open holdings are imported; cash operations are left out.
</>
)}
</span>
<input
type="file"
accept={
aiEnabled ? ".csv,.txt,.htm,.html,.tsv,.xml,.pdf" : ".csv,.txt,.htm,.html,.tsv,.xml"
aiEnabled
? ".csv,.txt,.htm,.html,.tsv,.xml,.pdf"
: ".csv,.txt,.htm,.html,.tsv,.xml,.xlsx"
}
disabled={busy || !settingsData || !validTimeZone}
className="hidden"
Expand Down
2 changes: 2 additions & 0 deletions apps/web/src/lib/xlsx-import.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
/** Largest .xlsx statement the import page accepts; XTB account workbooks are far smaller. */
export const XLSX_MAX_BYTES = 5 * 1024 * 1024;
91 changes: 91 additions & 0 deletions apps/web/src/server/xtb-import.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
import { and, eq } from "drizzle-orm";
import { xtbWorkbookToStatement, type ImportedExecution } from "@luxalgo/journal-importers";
import { db, executions } from "@/db";
import { requireValue } from "./api";
import { XLSX_MAX_BYTES } from "@/lib/xlsx-import";
import { executionHash } from "./ids";

/** Decode an uploaded base64 .xlsx on the server; the browser never parses the workbook. */
export const xlsxStatement = async (base64: string): Promise<string> => {
const tooLarge = "Excel uploads must be a .xlsx file of 5 MB or less.";
requireValue(base64.length <= Math.ceil(XLSX_MAX_BYTES / 3) * 4, tooLarge);
requireValue(/^[A-Za-z0-9+/]+={0,2}$/.test(base64), "This file is not an .xlsx workbook.");
const bytes = Buffer.from(base64, "base64");
requireValue(bytes.length <= XLSX_MAX_BYTES, tooLarge);
try {
return await xtbWorkbookToStatement(bytes);
} catch (error) {
// Reader and XTB layout errors describe the file, so they are the user's answer.
if (error instanceof Error) requireValue(false, error.message);
throw error;
}
};

const isXtbEntry = (fill: ImportedExecution): boolean =>
fill.importMetadata?.id === "entry" && Boolean(fill.importMetadata.group?.startsWith("xtb:"));

/** The same lot read from Open Positions and later from Closed Positions. */
const sameLot = (stored: ImportedExecution, incoming: ImportedExecution): boolean =>
stored.symbol === incoming.symbol &&
stored.side === incoming.side &&
stored.quantity.toPrecision(12) === incoming.quantity.toPrecision(12) &&
Math.abs(Date.parse(stored.executedAt) - Date.parse(incoming.executedAt)) < 1000 &&
Math.abs(stored.price - incoming.price) <=
1e-3 * Math.max(Math.abs(stored.price), Math.abs(incoming.price));

const describe = (fill: ImportedExecution) =>
`${fill.side} ${fill.quantity} ${fill.symbol} at ${fill.price} on ${fill.executedAt}`;

/**
* An XTB holding is stored as an entry fill keyed by its position id. When a
* later workbook closes that position, its entry must be the stored one, or the
* journal would keep a second, phantom open lot. A lot whose time or price
* differ only by rounding is the same fill and is not inserted again; a lot
* whose size or side changed (for example a partial close) is refused, because
* which part was closed cannot be known. Nothing is saved in that case.
*/
export const reconcileXtbHoldings = (
accountId: string,
incoming: ImportedExecution[],
): { executions: ImportedExecution[]; matched: number; problem: string | null } => {
const entries = incoming.filter(isXtbEntry);
if (!entries.length) return { executions: incoming, matched: 0, problem: null };
const stored = new Map<string, { fill: ImportedExecution; hash: string }>();
for (const row of db
.select()
.from(executions)
.where(and(eq(executions.accountId, accountId), eq(executions.source, "import")))
.all()) {
const importMetadata = row.importMetadataJson ? JSON.parse(row.importMetadataJson) : undefined;
const fill: ImportedExecution = {
symbol: row.symbol,
side: row.side as "buy" | "sell",
quantity: row.quantity,
price: row.price,
fee: row.fee,
executedAt: row.executedAt,
importMetadata,
};
if (isXtbEntry(fill)) stored.set(importMetadata.group, { fill, hash: row.contentHash });
}

const dropped = new Set<ImportedExecution>();
for (const entry of entries) {
const previous = stored.get(entry.importMetadata!.group!);
if (!previous || previous.hash === executionHash(entry)) continue;
if (!sameLot(previous.fill, entry)) {
const position = entry.importMetadata!.group!.split(":").at(-1);
return {
executions: incoming,
matched: 0,
problem: `XTB position ${position} was imported earlier as ${describe(previous.fill)}, but this file reports ${describe(entry)}. The position changed, for example by a partial close, so the journal cannot tell which part this file closes. Nothing was saved. Export one workbook whose date range covers both the opening and the close of this position, and import it into a new journal account.`,
};
}
dropped.add(entry);
}
return {
executions: incoming.filter((fill) => !dropped.has(fill)),
matched: dropped.size,
problem: null,
};
};
Loading
Loading