From 8dde5aa30259e810afd5fbd7de40912e82044ed1 Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sat, 24 Oct 2026 13:48:01 +0000 Subject: [PATCH] money: Revolut statement importer (CSV/Excel/PDF) with dedupe, dry run and start-balance removal --- scripts/revolut-import.ts | 337 ++++++++++++++++++++++ src/lib/server/revolut-statement.test.ts | 103 +++++++ src/lib/server/revolut-statement.ts | 343 +++++++++++++++++++++++ 3 files changed, 783 insertions(+) create mode 100644 scripts/revolut-import.ts create mode 100644 src/lib/server/revolut-statement.test.ts create mode 100644 src/lib/server/revolut-statement.ts diff --git a/scripts/revolut-import.ts b/scripts/revolut-import.ts new file mode 100644 index 0000000..bbd9389 --- /dev/null +++ b/scripts/revolut-import.ts @@ -0,0 +1,337 @@ +// Import a Revolut statement (all-time CSV, Excel or PDF export) into one tracker account, +// deduped against what's already there (bank-sync rows, Daisy's earlier imports, hand entries). +// +// bun --env-file=$HOME/.config/money-sync.env scripts/revolut-import.ts \ +// --file statement.csv --account Joint [options] +// (default) dry run: report new rows, duplicates, conflicts, resulting balances +// --product Current Revolut sub-account to import (Savings/pockets are reported, skipped) +// --currency EUR only this currency (other currency pockets are reported, skipped) +// --opening PDF only: the statement's opening balance (default 0) +// --list print every new row / conflict / tracker-only row +// --insert-conflicts also insert rows that fall in the already-covered period but match nothing +// --insert-conflicts-until YYYY-MM-DD same, but only conflicts up to that date (e.g. a row +// the old "Start balance" had absorbed on its own date) +// --transfer-pattern statement rows of type Transfer whose description matches get +// category 'transfer' (e.g. Personal <-> Joint legs) +// --tag-matched-transfers with --transfer-pattern: also set category 'transfer' on the +// existing tracker rows those statement rows matched +// --apply write (needs a money-schema backup in ~/backups from the last 2 hours) +// --drop-start-balance with --apply: delete the account's "Start balance" row(s), only if the +// statement opens at €0 and, after the import, the only difference +// to the synced bank balance is the tracker-only rows (hand entries +// the bank doesn't have). The sync's balance-adjustment row is then +// set to that difference (removed when €0), so the total stays equal +// to the bank balance. +// +// Matching: same amount, date within ±2 days (booking or started date), closest first, +// description overlap as tie-breaker. Inserted rows get source 'csv' and a stable +// external_id 'revolut-stmt:', so re-running never duplicates. +import { createHash } from 'node:crypto'; +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { homedir } from 'node:os'; +import postgres from 'postgres'; +import { + checkBalances, + parseRevolutCsv, + parseRevolutPdfLines, + planImport, + rowsFromTable, + selectRows, + statementExternalId, + type StatementRow, + type TrackerRow +} from '../src/lib/server/revolut-statement'; +import { BALANCE_ADJUSTMENT_ID, isOpeningBalance } from '../src/lib/server/bank-sync'; + +const args = process.argv.slice(2); +const value = (name: string) => { + const i = args.indexOf(name); + return i >= 0 ? args[i + 1] : undefined; +}; +const flag = (name: string) => args.includes(name); +const file = value('--file'); +const accountName = value('--account'); +if (!file || !accountName) + throw new Error('usage: --file --account '); +if (!process.env.DB_URL) throw new Error('DB_URL is not set'); +const apply = flag('--apply'); +const euro = (c: number | null | undefined) => (c == null ? '—' : `€${(c / 100).toFixed(2)}`); + +function xlsxTable(path: string): string[][] { + const read = (entry: string) => { + try { + return execFileSync('unzip', ['-p', path, entry], { maxBuffer: 1 << 28 }).toString('utf8'); + } catch { + return ''; + } + }; + const unxml = (s: string) => + s + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&/g, '&'); + const shared = [...read('xl/sharedStrings.xml').matchAll(/([\s\S]*?)<\/si>/g)].map((m) => + unxml([...m[1].matchAll(/]*>([\s\S]*?)<\/t>/g)].map((t) => t[1]).join('')) + ); + const sheet = read('xl/worksheets/sheet1.xml'); + const colIndex = (ref: string) => + [...ref.replace(/\d+/g, '')].reduce((n, ch) => n * 26 + ch.charCodeAt(0) - 64, 0) - 1; + return [...sheet.matchAll(/]*>([\s\S]*?)<\/row>/g)].map((row) => { + const cells: string[] = []; + for (const c of row[1].matchAll(/]*?)(?:\/>|>([\s\S]*?)<\/c>)/g)) { + const type = c[2].match(/t="(\w+)"/)?.[1]; + const v = + c[3]?.match(/([\s\S]*?)<\/v>/)?.[1] ?? c[3]?.match(/]*>([\s\S]*?)<\/t>/)?.[1] ?? ''; + cells[colIndex(c[1])] = type === 's' ? (shared[Number(v)] ?? '') : unxml(v); + } + return Array.from(cells, (x) => x ?? ''); + }); +} + +async function pdfLines(path: string): Promise { + const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs'); + const doc = await pdfjs.getDocument({ data: new Uint8Array(readFileSync(path)) }).promise; + const lines: string[] = []; + for (let p = 1; p <= doc.numPages; p++) { + const content = await (await doc.getPage(p)).getTextContent(); + const byY = new Map(); + for (const item of content.items as { str: string; transform: number[] }[]) { + const y = Math.round(item.transform[5]); + byY.set(y, [...(byY.get(y) ?? []), { x: item.transform[4], s: item.str }]); + } + for (const y of [...byY.keys()].sort((a, b) => b - a)) + lines.push( + byY + .get(y)! + .sort((a, b) => a.x - b.x) + .map((i) => i.s) + .join(' ') + ); + } + return lines; +} + +function loadStatement(path: string): StatementRow[] { + const lower = path.toLowerCase(); + if (lower.endsWith('.xlsx')) return rowsFromTable(xlsxTable(path)); + if (lower.endsWith('.csv') || lower.endsWith('.txt')) + return parseRevolutCsv(readFileSync(path, 'utf8')); + throw new Error('Use .csv, .xlsx or .pdf'); +} + +const all = file.toLowerCase().endsWith('.pdf') + ? parseRevolutPdfLines(await pdfLines(file), Math.round(Number(value('--opening') ?? 0) * 100)) + : loadStatement(file); +const { kept, skipped } = selectRows( + all, + value('--product') ?? 'Current', + value('--currency') ?? 'EUR' +); +// Chronological by booking date; file order breaks ties (it follows the running balance). +const rows = kept.slice().sort((a, b) => a.date.localeCompare(b.date) || a.line - b.line); +const balances = checkBalances(kept); + +const sql = postgres(process.env.DB_URL, { max: 2, connect_timeout: 10, onnotice: () => {} }); +try { + const [account] = await sql< + { + id: number; + user_id: string; + name: string; + opening_balance_cents: number; + bank_balance_cents: number | null; + bank_balance_at: Date | null; + }[] + >`select id, user_id, name, opening_balance_cents, bank_balance_cents, bank_balance_at + from money.accounts where lower(name) = lower(${accountName})`; + if (!account) throw new Error(`No tracker account named ${accountName}`); + const existing = await sql< + { + id: number; + date: string; + amount_cents: number; + description: string; + source: string; + external_id: string | null; + text: string; + }[] + >`select id, date::text, amount_cents, description, source, external_id, + concat_ws(' ', description, counterparty, raw->'creditor'->>'name', raw->'debtor'->>'name', + raw->>'remittance_information', raw->'revolut'->>'description') as text + from money.transactions where account_id = ${account.id} order by date, id`; + const startRows = existing.filter( + (t) => isOpeningBalance(t.description) && t.source === 'manual' + ); + const adjustment = existing.find((t) => t.external_id === BALANCE_ADJUSTMENT_ID); + const tracker: TrackerRow[] = existing + .filter((t) => !startRows.includes(t) && t !== adjustment) + .map((t) => ({ + id: t.id, + date: t.date, + amountCents: t.amount_cents, + description: t.description, + text: t.text, + source: t.source, + externalId: t.external_id + })); + const plan = planImport(rows, tracker); + const conflictsUntil = value('--insert-conflicts-until'); + const toInsert = flag('--insert-conflicts') + ? [...plan.fresh, ...plan.conflicts] + : conflictsUntil + ? [...plan.fresh, ...plan.conflicts.filter((r) => r.date <= conflictsUntil)] + : plan.fresh; + const transferRe = value('--transfer-pattern') + ? new RegExp(value('--transfer-pattern')!, 'i') + : null; + const isTransfer = (r: StatementRow) => + !!transferRe && /transfer/i.test(r.type) && transferRe.test(r.description); + + const today = new Date().toISOString().slice(0, 10); + const sum = (xs: { amountCents: number; date: string }[]) => + xs.filter((x) => x.date <= today).reduce((s, x) => s + x.amountCents, 0); + const trackedNow = + account.opening_balance_cents + + sum(tracker) + + sum(startRows.map((t) => ({ amountCents: t.amount_cents, date: t.date }))) + + (adjustment?.amount_cents ?? 0); + const afterNoStart = account.opening_balance_cents + sum(tracker) + sum(toInsert); + const bank = account.bank_balance_cents; + const residual = bank == null ? null : bank - afterNoStart; + const trackerOnlyCents = sum(plan.trackerOnly); + const dropOk = balances.openingCents === 0 && residual !== null && residual === -trackerOnlyCents; + const boundary = plan.coverageFrom + ? (rows.filter((r) => r.date < plan.coverageFrom! && r.balanceCents != null).at(-1) + ?.balanceCents ?? 0) + : null; + + console.log( + `Statement ${file}: ${all.length} rows; importing ${account.name} = ${value('--product') ?? 'Current'} ${value('--currency') ?? 'EUR'}: ${kept.length} completed rows ${rows[0]?.date ?? ''}..${rows.at(-1)?.date ?? ''}` + ); + const skippedBy = new Map(); + for (const r of skipped) { + const key = `${r.product}/${r.currency}/${r.state}`; + const e = skippedBy.get(key) ?? { n: 0, cents: 0 }; + skippedBy.set(key, { n: e.n + 1, cents: e.cents + r.amountCents }); + } + for (const [k, v] of skippedBy) console.log(` skipped ${k}: ${v.n} rows (net ${euro(v.cents)})`); + console.log( + ` statement opening ${euro(balances.openingCents)}, closing ${euro(balances.closingCents)}, running-balance breaks ${balances.breaks.length}` + ); + console.log( + `Tracker ${account.name} (#${account.id}): ${tracker.length} rows from ${plan.coverageFrom ?? '-'}; start balance rows ${startRows.map((t) => `#${t.id} ${t.date} ${euro(t.amount_cents)}`).join(', ') || 'none'}; adjustment ${euro(adjustment?.amount_cents)}` + ); + console.log( + ` statement balance just before ${plan.coverageFrom}: ${euro(boundary)} (tracker start + adjustment: ${euro(sum(startRows.map((t) => ({ amountCents: t.amount_cents, date: t.date }))) + (adjustment?.amount_cents ?? 0))})` + ); + console.log( + `Plan: new ${plan.fresh.length} (${euro(sum(plan.fresh))}), duplicates ${plan.duplicates.length} (${plan.duplicates.filter((d) => !d.descMatch).length} with differing description), conflicts ${plan.conflicts.length} (${euro(sum(plan.conflicts))}), tracker-only ${plan.trackerOnly.length} (${euro(sum(plan.trackerOnly))})` + ); + console.log( + `Balances: bank ${euro(bank)} (synced ${account.bank_balance_at?.toISOString() ?? 'never'}), tracker now ${euro(trackedNow)}, after import without start balance/adjustment ${euro(afterNoStart)}, gap ${euro(residual)}${residual !== null && residual === -trackerOnlyCents ? ' = tracker-only rows, OK to drop the start balance' : ` (tracker-only rows ${euro(trackerOnlyCents)}; start balance must stay)`}${transferRe ? `; transfers matching pattern: ${rows.filter(isTransfer).length}` : ''}` + ); + if (flag('--list') || !apply) { + const show = ( + label: string, + xs: { + date: string; + amountCents: number; + description: string; + id?: number; + source?: string; + }[], + limit: number + ) => { + if (!xs.length) return; + console.log(` ${label}:`); + for (const x of xs.slice(0, flag('--list') ? Infinity : limit)) + console.log( + ` ${x.date} ${euro(x.amountCents).padStart(10)} ${x.description}${x.id ? ` (#${x.id} ${x.source})` : ''}` + ); + if (!flag('--list') && xs.length > limit) + console.log(` … ${xs.length - limit} more (--list)`); + }; + show('conflicts (statement only, in covered period)', plan.conflicts, 30); + show('tracker-only rows (covered period, no statement row)', plan.trackerOnly, 30); + show( + 'duplicates with differing description', + plan.duplicates + .filter((d) => !d.descMatch) + .map((d) => ({ + ...d.row, + description: `${d.row.description} == ${d.match.description}` + })), + 10 + ); + show('new rows', plan.fresh, 5); + } + if (!apply) { + console.log('Dry run: nothing written. Re-run with --apply (after a backup) to import.'); + } else { + const dir = `${homedir()}/backups`; + const fresh = readdirSync(dir).some( + (f) => + f.startsWith('money-before-') && + f.endsWith('.dump') && + Date.now() - statSync(`${dir}/${f}`).mtimeMs < 2 * 3600_000 + ); + if (!fresh) + throw new Error(`No money-before-*.dump in ${dir} from the last 2 hours; back up first`); + const drop = flag('--drop-start-balance'); + if (drop && !dropOk) + throw new Error( + `Refusing --drop-start-balance: statement opening ${euro(balances.openingCents)}, gap ${euro(residual)} vs tracker-only rows ${euro(trackerOnlyCents)}` + ); + const hash = (s: string) => createHash('sha256').update(s).digest('hex'); + await sql.begin(async (q) => { + let inserted = 0; + for (const r of toInsert) { + const res = await q` + insert into money.transactions + (user_id, account_id, date, value_date, description, amount_cents, external_id, source, booking_status, category, raw) + values (${account.user_id}, ${account.id}, ${r.date}, ${r.startedDate}, ${r.description.slice(0, 500)}, + ${r.amountCents}, ${statementExternalId(r, hash)}, 'csv', 'BOOK', ${isTransfer(r) ? 'transfer' : null}, + ${q.json({ revolut: { type: r.type, product: r.product, started: r.startedDate, completed: r.completedAt, description: r.description, fee_cents: r.feeCents, currency: r.currency, balance_cents: r.balanceCents, file_line: r.line } })}) + on conflict do nothing`; + inserted += res.count; + } + console.log(`Inserted ${inserted} rows into ${account.name}.`); + if (transferRe && flag('--tag-matched-transfers')) { + const ids = plan.duplicates.filter((d) => isTransfer(d.row)).map((d) => d.match.id); + const tagged = ids.length + ? await q`update money.transactions set category = 'transfer' + where account_id = ${account.id} and id in ${q(ids)} and category is distinct from 'transfer'` + : { count: 0 }; + console.log(`Tagged ${tagged.count} existing transfer rows.`); + } + if (drop) { + for (const t of startRows) + await q`delete from money.transactions where id = ${t.id} and account_id = ${account.id}`; + const needed = residual!; + if (adjustment && needed === 0) + await q`delete from money.transactions where id = ${adjustment.id}`; + else if (adjustment) + await q`update money.transactions set amount_cents = ${needed} where id = ${adjustment.id}`; + else if (needed !== 0) + await q`insert into money.transactions + (user_id, account_id, date, description, amount_cents, external_id, source, booking_status) + values (${account.user_id}, ${account.id}, ${plan.coverageFrom ?? rows[0].date}, 'Bank balance adjustment', + ${needed}, ${BALANCE_ADJUSTMENT_ID}, 'enablebanking', 'ADJ')`; + const [check] = await q<{ total: number }[]>` + select (a.opening_balance_cents + coalesce(sum(t.amount_cents) filter (where t.date <= ${today}), 0))::int as total + from money.accounts a left join money.transactions t on t.account_id = a.id + where a.id = ${account.id} group by a.id`; + if (bank != null && check.total !== bank) + throw new Error(`Total would be ${euro(check.total)}, bank ${euro(bank)}; rolled back`); + console.log( + `Removed start balance ${startRows.map((t) => '#' + t.id).join(', ') || '-'}; adjustment ${euro(adjustment?.amount_cents)} -> ${euro(needed)}; total ${euro(check.total)} = bank.` + ); + } + }); + } +} finally { + await sql.end(); +} diff --git a/src/lib/server/revolut-statement.test.ts b/src/lib/server/revolut-statement.test.ts new file mode 100644 index 0000000..3b9fc34 --- /dev/null +++ b/src/lib/server/revolut-statement.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest'; +import { + checkBalances, + parseAmount, + parseRevolutCsv, + parseRevolutPdfLines, + planImport, + selectRows, + type TrackerRow +} from './revolut-statement'; + +const CSV = `Type,Product,Started Date,Completed Date,Description,Amount,Fee,Currency,State,Balance +Transfer,Current,2024-06-30 15:09:04,2024-06-30 15:09:04,Transfer from STAN RUNGE,100.00,0.00,EUR,COMPLETED,100.00 +Card Payment,Current,2024-07-01 19:12:32,2024-07-02 13:57:17,"Wok, Baldakijn",-72.10,0.00,EUR,COMPLETED,27.90 +Card Payment,Current,2024-07-03 10:00:00,,Shop,-5.00,0.00,EUR,REVERTED, +Exchange,Current,2024-07-04 10:00:00,2024-07-04 10:00:01,Exchanged to USD,-10.00,0.20,EUR,COMPLETED,17.70 +Exchange,Current,2024-07-04 10:00:00,2024-07-04 10:00:01,Exchanged from EUR,10.80,0.00,USD,COMPLETED,10.80 +CASHBACK,Savings,2024-08-09 18:56:54,2024-08-10 08:42:12,Plan Cashback,0.01,0.00,EUR,COMPLETED,0.01 +Card Payment,Current,2025-08-02 10:00:00,2025-08-03 09:00:00,Albert Heijn 1234,-12.34,0.00,EUR,COMPLETED,5.36 +Card Payment,Current,2025-08-05 10:00:00,2025-08-05 11:00:00,Kiosk,-3.50,0.00,EUR,COMPLETED,1.86 +`; + +describe('revolut statement', () => { + it('parses amounts in both notations', () => { + expect(parseAmount('1,234.56')).toBe(123456); + expect(parseAmount('-1.234,56')).toBe(-123456); + expect(parseAmount('€ 12,50')).toBe(1250); + expect(parseAmount('')).toBeNull(); + }); + + it('parses the CSV export, nets fees and keeps only completed Current EUR rows', () => { + const rows = parseRevolutCsv(CSV); + expect(rows).toHaveLength(8); + expect(rows[1]).toMatchObject({ + date: '2024-07-02', + startedDate: '2024-07-01', + description: 'Wok, Baldakijn', + amountCents: -7210 + }); + const { kept, skipped } = selectRows(rows); + expect(kept.map((r) => r.amountCents)).toEqual([10000, -7210, -1020, -1234, -350]); + expect(skipped.map((r) => `${r.product}/${r.currency}/${r.state}`)).toEqual([ + 'Current/EUR/REVERTED', + 'Current/USD/COMPLETED', + 'Savings/EUR/COMPLETED' + ]); + expect(checkBalances(kept)).toMatchObject({ openingCents: 0, closingCents: 186, breaks: [] }); + }); + + it('reads Dutch semicolon headers', () => { + const rows = parseRevolutCsv( + 'Soort;Product;Begindatum;Datum voltooid;Beschrijving;Bedrag;Kosten;Valuta;Status;Saldo\nKaartbetaling;Current;2024-01-02 10:00:00;2024-01-03 10:00:00;Jumbo;-3,50;0,00;EUR;VOLTOOID;10,00\n' + ); + expect(rows[0]).toMatchObject({ date: '2024-01-03', amountCents: -350, balanceCents: 1000 }); + }); + + it('parses PDF statement lines using the running balance for direction', () => { + const rows = parseRevolutPdfLines([ + 'EUR Statement', + '30 Jun 2024 Transfer from STAN RUNGE €100.00 €100.00', + 'Jul 2, 2024 Wok Van Baldakijn €72.10 €27.90', + '2 jul. 2024 Something €1.234,00 €1.261,90', + 'Balance summary €0.00 €27.90' + ]); + expect(rows.map((r) => [r.date, r.amountCents])).toEqual([ + ['2024-06-30', 10000], + ['2024-07-02', -7210], + ['2024-07-02', 123400] + ]); + }); + + it('plans an import: duplicates within ±2 days, older unmatched rows are new, newer are conflicts', () => { + const { kept } = selectRows(parseRevolutCsv(CSV)); + const tracker: TrackerRow[] = [ + { + id: 1, + date: '2025-08-02', + amountCents: -1234, + description: 'Groceries', + text: 'Groceries ALBERT HEIJN 1234', + source: 'manual', + externalId: 'x' + }, + { + id: 2, + date: '2025-08-04', + amountCents: -999, + description: 'Cash', + text: 'Cash', + source: 'manual', + externalId: null + } + ]; + const plan = planImport(kept, tracker); + expect(plan.coverageFrom).toBe('2025-08-02'); + expect(plan.duplicates.map((d) => [d.row.amountCents, d.match.id, d.descMatch])).toEqual([ + [-1234, 1, true] + ]); + expect(plan.fresh.map((r) => r.amountCents)).toEqual([10000, -7210, -1020]); + expect(plan.conflicts.map((r) => r.amountCents)).toEqual([-350]); + expect(plan.trackerOnly.map((t) => t.id)).toEqual([2]); + }); +}); diff --git a/src/lib/server/revolut-statement.ts b/src/lib/server/revolut-statement.ts new file mode 100644 index 0000000..702a90a --- /dev/null +++ b/src/lib/server/revolut-statement.ts @@ -0,0 +1,343 @@ +// Revolut account statement parsing + import planning (pure; DB work is in scripts/revolut-import.ts). +// Handles the app's CSV export (English or Dutch headers, , or ; separated), the Excel export +// (rows passed in as a string table) and, defensively, the text of the PDF statement. + +export type StatementRow = { + /** Booking date (completed date, else started date), YYYY-MM-DD. */ + date: string; + startedDate: string; + completedAt: string; + type: string; + product: string; + description: string; + /** Net balance change: amount minus fee. */ + amountCents: number; + feeCents: number; + currency: string; + state: string; + balanceCents: number | null; + /** Position in the file, for stable ordering. */ + line: number; +}; + +const HEADERS: Record = { + type: ['type', 'soort'], + product: ['product'], + started: ['started date', 'begindatum', 'startdatum', 'datum gestart'], + completed: ['completed date', 'datum voltooid', 'voltooid', 'einddatum'], + description: ['description', 'beschrijving', 'omschrijving'], + amount: ['amount', 'bedrag'], + fee: ['fee', 'kosten', 'vergoeding'], + currency: ['currency', 'valuta'], + state: ['state', 'status'], + balance: ['balance', 'saldo'] +}; + +/** Parses "1,234.56", "-1.234,56", "€ 12,50", "1234" into cents. */ +export function parseAmount(raw: string | undefined | null): number | null { + if (raw == null) return null; + let s = String(raw) + .replace(/[€$£\s\u00a0]|EUR|USD/g, '') + .replace(/^\((.*)\)$/, '-$1') + .replace(/[−–]/g, '-'); + if (!s || s === '-') return null; + const lastComma = s.lastIndexOf(','); + const lastDot = s.lastIndexOf('.'); + if (lastComma > lastDot) s = s.replace(/\./g, '').replace(',', '.'); + else s = s.replace(/,/g, ''); + if (!/^[-+]?\d+(\.\d+)?$/.test(s)) return null; + return Math.round(Number(s) * 100); +} + +/** RFC 4180-ish CSV parser; detects , or ; from the header line. */ +export function parseCsv(text: string): string[][] { + text = text.replace(/^\uFEFF/, ''); + const firstLine = text.slice(0, text.indexOf('\n') >= 0 ? text.indexOf('\n') : text.length); + const sep = + (firstLine.match(/;/g)?.length ?? 0) > (firstLine.match(/,/g)?.length ?? 0) ? ';' : ','; + const rows: string[][] = []; + let row: string[] = []; + let field = ''; + let quoted = false; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + if (quoted) { + if (c === '"' && text[i + 1] === '"') { + field += '"'; + i++; + } else if (c === '"') quoted = false; + else field += c; + } else if (c === '"') quoted = true; + else if (c === sep) { + row.push(field); + field = ''; + } else if (c === '\n' || c === '\r') { + if (c === '\r' && text[i + 1] === '\n') i++; + row.push(field); + if (row.some((f) => f !== '')) rows.push(row); + row = []; + field = ''; + } else field += c; + } + row.push(field); + if (row.some((f) => f !== '')) rows.push(row); + return rows; +} + +function isoDay(value: string): string { + const v = value.trim(); + let m = v.match(/^(\d{4})-(\d{2})-(\d{2})/); + if (m) return `${m[1]}-${m[2]}-${m[3]}`; + m = v.match(/^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})/); + if (m) return `${m[3]}-${m[2].padStart(2, '0')}-${m[1].padStart(2, '0')}`; + // Excel serial day number + if (/^\d{5}(\.\d+)?$/.test(v)) { + const d = new Date(Math.round((Number(v) - 25569) * 86400) * 1000); + return d.toISOString().slice(0, 10); + } + return ''; +} + +/** Turns a header + rows table (CSV or Excel) into statement rows. */ +export function rowsFromTable(table: string[][]): StatementRow[] { + const header = (table[0] ?? []).map((h) => h.trim().toLowerCase()); + const col = (key: string) => header.findIndex((h) => HEADERS[key].includes(h)); + const idx = Object.fromEntries(Object.keys(HEADERS).map((k) => [k, col(k)])); + for (const key of ['started', 'description', 'amount']) + if (idx[key] < 0) throw new Error(`Not a Revolut statement: no "${HEADERS[key][0]}" column`); + const get = (r: string[], key: string) => (idx[key] >= 0 ? (r[idx[key]] ?? '').trim() : ''); + return table.slice(1).flatMap((r, i) => { + const amount = parseAmount(get(r, 'amount')); + const started = isoDay(get(r, 'started')); + if (amount == null || !started) return []; + const fee = parseAmount(get(r, 'fee')) ?? 0; + const completedAt = get(r, 'completed'); + return [ + { + date: isoDay(completedAt) || started, + startedDate: started, + completedAt, + type: get(r, 'type'), + product: get(r, 'product') || 'Current', + description: get(r, 'description'), + amountCents: amount - fee, + feeCents: fee, + currency: (get(r, 'currency') || 'EUR').toUpperCase(), + state: (get(r, 'state') || 'COMPLETED').toUpperCase(), + balanceCents: parseAmount(get(r, 'balance')), + line: i + 2 + } + ]; + }); +} + +export function parseRevolutCsv(text: string): StatementRow[] { + return rowsFromTable(parseCsv(text)); +} + +const MONTHS: Record = { + jan: 1, + feb: 2, + mar: 3, + mrt: 3, + apr: 4, + may: 5, + mei: 5, + jun: 6, + jul: 7, + aug: 8, + sep: 9, + oct: 10, + okt: 10, + nov: 11, + dec: 12 +}; +const PDF_DATE = + /^(?:(\d{1,2})\s+([a-z]{3})[a-z]*\.?\s+(\d{4})|([a-z]{3})[a-z]*\.?\s+(\d{1,2}),?\s+(\d{4}))\s+/i; +const MONEY = /[-−]?€\s?[\d.,]+|[-−]?[\d.,]+\s?€/g; + +/** + * Best-effort parser for the text lines of a Revolut PDF statement ("Date Description + * Money out Money in Balance"). In/out is taken from the balance change, so every + * transaction line must end in its running balance; `openingCents` is the statement's + * opening balance (0 for an all-time statement). + */ +export function parseRevolutPdfLines(lines: string[], openingCents = 0): StatementRow[] { + const out: StatementRow[] = []; + let balance = openingCents; + lines.forEach((raw, i) => { + const line = raw.replace(/\s+/g, ' ').trim(); + const m = line.match(PDF_DATE); + if (!m) return; + const day = Number(m[1] ?? m[5]); + const month = MONTHS[(m[2] ?? m[4]).toLowerCase()]; + const year = Number(m[3] ?? m[6]); + if (!month || !day) return; + const rest = line.slice(m[0].length); + const money = rest.match(MONEY); + if (!money || money.length < 2) return; + const next = parseAmount(money.at(-1)!.replace('−', '-')); + const amount = parseAmount(money.at(-2)!.replace('−', '-')); + if (next == null || amount == null) return; + const description = rest.slice(0, rest.indexOf(money[0])).trim(); + const delta = next - balance; + if (Math.abs(delta) !== Math.abs(amount)) return; // not a transaction line (or out of order) + const date = `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`; + out.push({ + date, + startedDate: date, + completedAt: date, + type: '', + product: 'Current', + description, + amountCents: delta, + feeCents: 0, + currency: 'EUR', + state: 'COMPLETED', + balanceCents: next, + line: i + 1 + }); + balance = next; + }); + return out; +} + +/** Rows that move money in the selected sub-account (completed only, one product + currency). */ +export function selectRows(rows: StatementRow[], product = 'Current', currency = 'EUR') { + const kept = rows.filter( + (r) => + r.state === 'COMPLETED' && + r.product.toLowerCase() === product.toLowerCase() && + r.currency === currency + ); + const skipped = rows.filter((r) => !kept.includes(r)); + return { kept, skipped }; +} + +/** Checks the running balance column; returns the balance before the first row. */ +export function checkBalances(rows: StatementRow[]) { + const breaks: StatementRow[] = []; + let prev: number | null = null; + for (const r of rows) { + if (r.balanceCents == null) continue; + if (prev != null && prev + r.amountCents !== r.balanceCents) breaks.push(r); + prev = r.balanceCents; + } + const first = rows.find((r) => r.balanceCents != null); + return { + openingCents: first ? first.balanceCents! - first.amountCents : null, + closingCents: prev, + breaks + }; +} + +export type TrackerRow = { + id: number; + date: string; + amountCents: number; + description: string; + /** Description, counterparty and raw bank names/remittance, for matching. */ + text: string; + source: string; + externalId: string | null; +}; + +const STOP = new Set([ + 'transfer', + 'from', + 'to', + 'payment', + 'the', + 'and', + 'van', + 'naar', + 'aan', + 'bv', + 'b.v', + 'eur' +]); +export function words(text: string) { + return new Set( + text + .toLowerCase() + .normalize('NFKD') + .replace(/[^a-z0-9 ]+/g, ' ') + .split(/\s+/) + .filter((w) => w.length > 2 && !STOP.has(w)) + ); +} + +function days(a: string, b: string) { + return Math.abs(Date.parse(a + 'T00:00:00Z') - Date.parse(b + 'T00:00:00Z')) / 86_400_000; +} + +export type ImportPlan = { + coverageFrom: string | null; + fresh: StatementRow[]; + duplicates: { row: StatementRow; match: TrackerRow; descMatch: boolean }[]; + /** In the period the tracker already covers, but no tracker row matches. */ + conflicts: StatementRow[]; + /** Tracker rows in the covered period that no statement row matched. */ + trackerOnly: TrackerRow[]; +}; + +/** + * Matches statement rows to tracker rows: same amount, booking or started date within + * `maxDays`, closest date first, description overlap breaking ties. Unmatched statement + * rows before the tracker's first real row are new; unmatched ones after it are conflicts. + */ +export function planImport( + rows: StatementRow[], + tracker: TrackerRow[], + { maxDays = 2, coverageFrom }: { maxDays?: number; coverageFrom?: string | null } = {} +): ImportPlan { + const from = coverageFrom ?? tracker.map((t) => t.date).sort()[0] ?? null; + const trackerWords = new Map(tracker.map((t) => [t.id, words(t.text)])); + const pairs: { r: number; t: TrackerRow; score: number; desc: boolean }[] = []; + const byAmount = new Map(); + for (const t of tracker) byAmount.set(t.amountCents, [...(byAmount.get(t.amountCents) ?? []), t]); + rows.forEach((row, r) => { + const w = words(row.description); + for (const t of byAmount.get(row.amountCents) ?? []) { + const diff = Math.min(days(row.date, t.date), days(row.startedDate, t.date)); + if (diff > maxDays) continue; + const tw = trackerWords.get(t.id)!; + const desc = [...w].some((x) => tw.has(x)); + pairs.push({ r, t, score: diff * 10 - (desc ? 5 : 0) + days(row.date, t.date) * 0.1, desc }); + } + }); + pairs.sort((a, b) => a.score - b.score || a.r - b.r || a.t.id - b.t.id); + const usedRows = new Set(); + const usedTracker = new Set(); + const duplicates: ImportPlan['duplicates'] = []; + for (const p of pairs) { + if (usedRows.has(p.r) || usedTracker.has(p.t.id)) continue; + usedRows.add(p.r); + usedTracker.add(p.t.id); + duplicates.push({ row: rows[p.r], match: p.t, descMatch: p.desc }); + } + const unmatched = rows.filter((_, i) => !usedRows.has(i)); + return { + coverageFrom: from, + fresh: unmatched.filter((r) => !from || r.date < from), + duplicates, + conflicts: unmatched.filter((r) => from && r.date >= from), + trackerOnly: tracker.filter((t) => !usedTracker.has(t.id) && (!from || t.date >= from)) + }; +} + +/** Stable id so re-running an import never inserts a row twice. */ +export function statementExternalId(row: StatementRow, hash: (s: string) => string) { + return ( + 'revolut-stmt:' + + hash( + [ + row.completedAt || row.startedDate, + row.startedDate, + row.amountCents, + row.description, + row.balanceCents ?? '' + ].join('|') + ).slice(0, 24) + ); +}