money: Revolut statement importer (CSV/Excel/PDF) with dedupe, dry run and start-balance removal

src/lib/server/revolut-statement.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect, it } from 'vitest';
+import {
+	checkBalances,
+	parseAmount,
+	parseRevolutCsv,
+	parseRevolutPdfLines,
+	planImport,
+	selectRows,
+	type TrackerRow
+} from './revolut-statement';
+
+const CSV = `Type,Product,Started Date,Completed Date,Description,Amount,Fee,Currency,State,Balance
+Transfer,Current,2024-06-30 15:09:04,2024-06-30 15:09:04,Transfer from STAN RUNGE,100.00,0.00,EUR,COMPLETED,100.00
+Card Payment,Current,2024-07-01 19:12:32,2024-07-02 13:57:17,"Wok, Baldakijn",-72.10,0.00,EUR,COMPLETED,27.90
+Card Payment,Current,2024-07-03 10:00:00,,Shop,-5.00,0.00,EUR,REVERTED,
+Exchange,Current,2024-07-04 10:00:00,2024-07-04 10:00:01,Exchanged to USD,-10.00,0.20,EUR,COMPLETED,17.70
+Exchange,Current,2024-07-04 10:00:00,2024-07-04 10:00:01,Exchanged from EUR,10.80,0.00,USD,COMPLETED,10.80
+CASHBACK,Savings,2024-08-09 18:56:54,2024-08-10 08:42:12,Plan Cashback,0.01,0.00,EUR,COMPLETED,0.01
+Card Payment,Current,2025-08-02 10:00:00,2025-08-03 09:00:00,Albert Heijn 1234,-12.34,0.00,EUR,COMPLETED,5.36
+Card Payment,Current,2025-08-05 10:00:00,2025-08-05 11:00:00,Kiosk,-3.50,0.00,EUR,COMPLETED,1.86
+`;
+
+describe('revolut statement', () => {
+	it('parses amounts in both notations', () => {
+		expect(parseAmount('1,234.56')).toBe(123456);
+		expect(parseAmount('-1.234,56')).toBe(-123456);
+		expect(parseAmount('€ 12,50')).toBe(1250);
+		expect(parseAmount('')).toBeNull();
+	});
+
+	it('parses the CSV export, nets fees and keeps only completed Current EUR rows', () => {
+		const rows = parseRevolutCsv(CSV);
+		expect(rows).toHaveLength(8);
+		expect(rows[1]).toMatchObject({
+			date: '2024-07-02',
+			startedDate: '2024-07-01',
+			description: 'Wok, Baldakijn',
+			amountCents: -7210
+		});
+		const { kept, skipped } = selectRows(rows);
+		expect(kept.map((r) => r.amountCents)).toEqual([10000, -7210, -1020, -1234, -350]);
+		expect(skipped.map((r) => `${r.product}/${r.currency}/${r.state}`)).toEqual([
+			'Current/EUR/REVERTED',
+			'Current/USD/COMPLETED',
+			'Savings/EUR/COMPLETED'
+		]);
+		expect(checkBalances(kept)).toMatchObject({ openingCents: 0, closingCents: 186, breaks: [] });
+	});
+
+	it('reads Dutch semicolon headers', () => {
+		const rows = parseRevolutCsv(
+			'Soort;Product;Begindatum;Datum voltooid;Beschrijving;Bedrag;Kosten;Valuta;Status;Saldo\nKaartbetaling;Current;2024-01-02 10:00:00;2024-01-03 10:00:00;Jumbo;-3,50;0,00;EUR;VOLTOOID;10,00\n'
+		);
+		expect(rows[0]).toMatchObject({ date: '2024-01-03', amountCents: -350, balanceCents: 1000 });
+	});
+
+	it('parses PDF statement lines using the running balance for direction', () => {
+		const rows = parseRevolutPdfLines([
+			'EUR Statement',
+			'30 Jun 2024 Transfer from STAN RUNGE €100.00 €100.00',
+			'Jul 2, 2024 Wok Van Baldakijn €72.10 €27.90',
+			'2 jul. 2024 Something €1.234,00 €1.261,90',
+			'Balance summary €0.00 €27.90'
+		]);
+		expect(rows.map((r) => [r.date, r.amountCents])).toEqual([
+			['2024-06-30', 10000],
+			['2024-07-02', -7210],
+			['2024-07-02', 123400]
+		]);
+	});
+
+	it('plans an import: duplicates within ±2 days, older unmatched rows are new, newer are conflicts', () => {
+		const { kept } = selectRows(parseRevolutCsv(CSV));
+		const tracker: TrackerRow[] = [
+			{
+				id: 1,
+				date: '2025-08-02',
+				amountCents: -1234,
+				description: 'Groceries',
+				text: 'Groceries ALBERT HEIJN 1234',
+				source: 'manual',
+				externalId: 'x'
+			},
+			{
+				id: 2,
+				date: '2025-08-04',
+				amountCents: -999,
+				description: 'Cash',
+				text: 'Cash',
+				source: 'manual',
+				externalId: null
+			}
+		];
+		const plan = planImport(kept, tracker);
+		expect(plan.coverageFrom).toBe('2025-08-02');
+		expect(plan.duplicates.map((d) => [d.row.amountCents, d.match.id, d.descMatch])).toEqual([
+			[-1234, 1, true]
+		]);
+		expect(plan.fresh.map((r) => r.amountCents)).toEqual([10000, -7210, -1020]);
+		expect(plan.conflicts.map((r) => r.amountCents)).toEqual([-350]);
+		expect(plan.trackerOnly.map((t) => t.id)).toEqual([2]);
+	});
+});

src/lib/server/revolut-statement.ts

@@ -0,0 +1,343 @@
+// Revolut account statement parsing + import planning (pure; DB work is in scripts/revolut-import.ts).
+// Handles the app's CSV export (English or Dutch headers, , or ; separated), the Excel export
+// (rows passed in as a string table) and, defensively, the text of the PDF statement.
+
+export type StatementRow = {
+	/** Booking date (completed date, else started date), YYYY-MM-DD. */
+	date: string;
+	startedDate: string;
+	completedAt: string;
+	type: string;
+	product: string;
+	description: string;
+	/** Net balance change: amount minus fee. */
+	amountCents: number;
+	feeCents: number;
+	currency: string;
+	state: string;
+	balanceCents: number | null;
+	/** Position in the file, for stable ordering. */
+	line: number;
+};
+
+const HEADERS: Record<string, string[]> = {
+	type: ['type', 'soort'],
+	product: ['product'],
+	started: ['started date', 'begindatum', 'startdatum', 'datum gestart'],
+	completed: ['completed date', 'datum voltooid', 'voltooid', 'einddatum'],
+	description: ['description', 'beschrijving', 'omschrijving'],
+	amount: ['amount', 'bedrag'],
+	fee: ['fee', 'kosten', 'vergoeding'],
+	currency: ['currency', 'valuta'],
+	state: ['state', 'status'],
+	balance: ['balance', 'saldo']
+};
+
+/** Parses "1,234.56", "-1.234,56", "€ 12,50", "1234" into cents. */
+export function parseAmount(raw: string | undefined | null): number | null {
+	if (raw == null) return null;
+	let s = String(raw)
+		.replace(/[€$£\s\u00a0]|EUR|USD/g, '')
+		.replace(/^\((.*)\)$/, '-$1')
+		.replace(/[−–]/g, '-');
+	if (!s || s === '-') return null;
+	const lastComma = s.lastIndexOf(',');
+	const lastDot = s.lastIndexOf('.');
+	if (lastComma > lastDot) s = s.replace(/\./g, '').replace(',', '.');
+	else s = s.replace(/,/g, '');
+	if (!/^[-+]?\d+(\.\d+)?$/.test(s)) return null;
+	return Math.round(Number(s) * 100);
+}
+
+/** RFC 4180-ish CSV parser; detects , or ; from the header line. */
+export function parseCsv(text: string): string[][] {
+	text = text.replace(/^\uFEFF/, '');
+	const firstLine = text.slice(0, text.indexOf('\n') >= 0 ? text.indexOf('\n') : text.length);
+	const sep =
+		(firstLine.match(/;/g)?.length ?? 0) > (firstLine.match(/,/g)?.length ?? 0) ? ';' : ',';
+	const rows: string[][] = [];
+	let row: string[] = [];
+	let field = '';
+	let quoted = false;
+	for (let i = 0; i < text.length; i++) {
+		const c = text[i];
+		if (quoted) {
+			if (c === '"' && text[i + 1] === '"') {
+				field += '"';
+				i++;
+			} else if (c === '"') quoted = false;
+			else field += c;
+		} else if (c === '"') quoted = true;
+		else if (c === sep) {
+			row.push(field);
+			field = '';
+		} else if (c === '\n' || c === '\r') {
+			if (c === '\r' && text[i + 1] === '\n') i++;
+			row.push(field);
+			if (row.some((f) => f !== '')) rows.push(row);
+			row = [];
+			field = '';
+		} else field += c;
+	}
+	row.push(field);
+	if (row.some((f) => f !== '')) rows.push(row);
+	return rows;
+}
+
+function isoDay(value: string): string {
+	const v = value.trim();
+	let m = v.match(/^(\d{4})-(\d{2})-(\d{2})/);
+	if (m) return `${m[1]}-${m[2]}-${m[3]}`;
+	m = v.match(/^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})/);
+	if (m) return `${m[3]}-${m[2].padStart(2, '0')}-${m[1].padStart(2, '0')}`;
+	// Excel serial day number
+	if (/^\d{5}(\.\d+)?$/.test(v)) {
+		const d = new Date(Math.round((Number(v) - 25569) * 86400) * 1000);
+		return d.toISOString().slice(0, 10);
+	}
+	return '';
+}
+
+/** Turns a header + rows table (CSV or Excel) into statement rows. */
+export function rowsFromTable(table: string[][]): StatementRow[] {
+	const header = (table[0] ?? []).map((h) => h.trim().toLowerCase());
+	const col = (key: string) => header.findIndex((h) => HEADERS[key].includes(h));
+	const idx = Object.fromEntries(Object.keys(HEADERS).map((k) => [k, col(k)]));
+	for (const key of ['started', 'description', 'amount'])
+		if (idx[key] < 0) throw new Error(`Not a Revolut statement: no "${HEADERS[key][0]}" column`);
+	const get = (r: string[], key: string) => (idx[key] >= 0 ? (r[idx[key]] ?? '').trim() : '');
+	return table.slice(1).flatMap((r, i) => {
+		const amount = parseAmount(get(r, 'amount'));
+		const started = isoDay(get(r, 'started'));
+		if (amount == null || !started) return [];
+		const fee = parseAmount(get(r, 'fee')) ?? 0;
+		const completedAt = get(r, 'completed');
+		return [
+			{
+				date: isoDay(completedAt) || started,
+				startedDate: started,
+				completedAt,
+				type: get(r, 'type'),
+				product: get(r, 'product') || 'Current',
+				description: get(r, 'description'),
+				amountCents: amount - fee,
+				feeCents: fee,
+				currency: (get(r, 'currency') || 'EUR').toUpperCase(),
+				state: (get(r, 'state') || 'COMPLETED').toUpperCase(),
+				balanceCents: parseAmount(get(r, 'balance')),
+				line: i + 2
+			}
+		];
+	});
+}
+
+export function parseRevolutCsv(text: string): StatementRow[] {
+	return rowsFromTable(parseCsv(text));
+}
+
+const MONTHS: Record<string, number> = {
+	jan: 1,
+	feb: 2,
+	mar: 3,
+	mrt: 3,
+	apr: 4,
+	may: 5,
+	mei: 5,
+	jun: 6,
+	jul: 7,
+	aug: 8,
+	sep: 9,
+	oct: 10,
+	okt: 10,
+	nov: 11,
+	dec: 12
+};
+const PDF_DATE =
+	/^(?:(\d{1,2})\s+([a-z]{3})[a-z]*\.?\s+(\d{4})|([a-z]{3})[a-z]*\.?\s+(\d{1,2}),?\s+(\d{4}))\s+/i;
+const MONEY = /[-−]?€\s?[\d.,]+|[-−]?[\d.,]+\s?€/g;
+
+/**
+ * Best-effort parser for the text lines of a Revolut PDF statement ("Date Description
+ * Money out Money in Balance"). In/out is taken from the balance change, so every
+ * transaction line must end in its running balance; `openingCents` is the statement's
+ * opening balance (0 for an all-time statement).
+ */
+export function parseRevolutPdfLines(lines: string[], openingCents = 0): StatementRow[] {
+	const out: StatementRow[] = [];
+	let balance = openingCents;
+	lines.forEach((raw, i) => {
+		const line = raw.replace(/\s+/g, ' ').trim();
+		const m = line.match(PDF_DATE);
+		if (!m) return;
+		const day = Number(m[1] ?? m[5]);
+		const month = MONTHS[(m[2] ?? m[4]).toLowerCase()];
+		const year = Number(m[3] ?? m[6]);
+		if (!month || !day) return;
+		const rest = line.slice(m[0].length);
+		const money = rest.match(MONEY);
+		if (!money || money.length < 2) return;
+		const next = parseAmount(money.at(-1)!.replace('−', '-'));
+		const amount = parseAmount(money.at(-2)!.replace('−', '-'));
+		if (next == null || amount == null) return;
+		const description = rest.slice(0, rest.indexOf(money[0])).trim();
+		const delta = next - balance;
+		if (Math.abs(delta) !== Math.abs(amount)) return; // not a transaction line (or out of order)
+		const date = `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
+		out.push({
+			date,
+			startedDate: date,
+			completedAt: date,
+			type: '',
+			product: 'Current',
+			description,
+			amountCents: delta,
+			feeCents: 0,
+			currency: 'EUR',
+			state: 'COMPLETED',
+			balanceCents: next,
+			line: i + 1
+		});
+		balance = next;
+	});
+	return out;
+}
+
+/** Rows that move money in the selected sub-account (completed only, one product + currency). */
+export function selectRows(rows: StatementRow[], product = 'Current', currency = 'EUR') {
+	const kept = rows.filter(
+		(r) =>
+			r.state === 'COMPLETED' &&
+			r.product.toLowerCase() === product.toLowerCase() &&
+			r.currency === currency
+	);
+	const skipped = rows.filter((r) => !kept.includes(r));
+	return { kept, skipped };
+}
+
+/** Checks the running balance column; returns the balance before the first row. */
+export function checkBalances(rows: StatementRow[]) {
+	const breaks: StatementRow[] = [];
+	let prev: number | null = null;
+	for (const r of rows) {
+		if (r.balanceCents == null) continue;
+		if (prev != null && prev + r.amountCents !== r.balanceCents) breaks.push(r);
+		prev = r.balanceCents;
+	}
+	const first = rows.find((r) => r.balanceCents != null);
+	return {
+		openingCents: first ? first.balanceCents! - first.amountCents : null,
+		closingCents: prev,
+		breaks
+	};
+}
+
+export type TrackerRow = {
+	id: number;
+	date: string;
+	amountCents: number;
+	description: string;
+	/** Description, counterparty and raw bank names/remittance, for matching. */
+	text: string;
+	source: string;
+	externalId: string | null;
+};
+
+const STOP = new Set([
+	'transfer',
+	'from',
+	'to',
+	'payment',
+	'the',
+	'and',
+	'van',
+	'naar',
+	'aan',
+	'bv',
+	'b.v',
+	'eur'
+]);
+export function words(text: string) {
+	return new Set(
+		text
+			.toLowerCase()
+			.normalize('NFKD')
+			.replace(/[^a-z0-9 ]+/g, ' ')
+			.split(/\s+/)
+			.filter((w) => w.length > 2 && !STOP.has(w))
+	);
+}
+
+function days(a: string, b: string) {
+	return Math.abs(Date.parse(a + 'T00:00:00Z') - Date.parse(b + 'T00:00:00Z')) / 86_400_000;
+}
+
+export type ImportPlan = {
+	coverageFrom: string | null;
+	fresh: StatementRow[];
+	duplicates: { row: StatementRow; match: TrackerRow; descMatch: boolean }[];
+	/** In the period the tracker already covers, but no tracker row matches. */
+	conflicts: StatementRow[];
+	/** Tracker rows in the covered period that no statement row matched. */
+	trackerOnly: TrackerRow[];
+};
+
+/**
+ * Matches statement rows to tracker rows: same amount, booking or started date within
+ * `maxDays`, closest date first, description overlap breaking ties. Unmatched statement
+ * rows before the tracker's first real row are new; unmatched ones after it are conflicts.
+ */
+export function planImport(
+	rows: StatementRow[],
+	tracker: TrackerRow[],
+	{ maxDays = 2, coverageFrom }: { maxDays?: number; coverageFrom?: string | null } = {}
+): ImportPlan {
+	const from = coverageFrom ?? tracker.map((t) => t.date).sort()[0] ?? null;
+	const trackerWords = new Map(tracker.map((t) => [t.id, words(t.text)]));
+	const pairs: { r: number; t: TrackerRow; score: number; desc: boolean }[] = [];
+	const byAmount = new Map<number, TrackerRow[]>();
+	for (const t of tracker) byAmount.set(t.amountCents, [...(byAmount.get(t.amountCents) ?? []), t]);
+	rows.forEach((row, r) => {
+		const w = words(row.description);
+		for (const t of byAmount.get(row.amountCents) ?? []) {
+			const diff = Math.min(days(row.date, t.date), days(row.startedDate, t.date));
+			if (diff > maxDays) continue;
+			const tw = trackerWords.get(t.id)!;
+			const desc = [...w].some((x) => tw.has(x));
+			pairs.push({ r, t, score: diff * 10 - (desc ? 5 : 0) + days(row.date, t.date) * 0.1, desc });
+		}
+	});
+	pairs.sort((a, b) => a.score - b.score || a.r - b.r || a.t.id - b.t.id);
+	const usedRows = new Set<number>();
+	const usedTracker = new Set<number>();
+	const duplicates: ImportPlan['duplicates'] = [];
+	for (const p of pairs) {
+		if (usedRows.has(p.r) || usedTracker.has(p.t.id)) continue;
+		usedRows.add(p.r);
+		usedTracker.add(p.t.id);
+		duplicates.push({ row: rows[p.r], match: p.t, descMatch: p.desc });
+	}
+	const unmatched = rows.filter((_, i) => !usedRows.has(i));
+	return {
+		coverageFrom: from,
+		fresh: unmatched.filter((r) => !from || r.date < from),
+		duplicates,
+		conflicts: unmatched.filter((r) => from && r.date >= from),
+		trackerOnly: tracker.filter((t) => !usedTracker.has(t.id) && (!from || t.date >= from))
+	};
+}
+
+/** Stable id so re-running an import never inserts a row twice. */
+export function statementExternalId(row: StatementRow, hash: (s: string) => string) {
+	return (
+		'revolut-stmt:' +
+		hash(
+			[
+				row.completedAt || row.startedDate,
+				row.startedDate,
+				row.amountCents,
+				row.description,
+				row.balanceCents ?? ''
+			].join('|')
+		).slice(0, 24)
+	);
+}