money: Revolut statement importer (CSV/Excel/PDF) with dedupe, dry run and start-balance removal
All checks were successful
Deploy production / deploy (push) Successful in 2m34s
All checks were successful
Deploy production / deploy (push) Successful in 2m34s
This commit is contained in:
337
scripts/revolut-import.ts
Normal file
337
scripts/revolut-import.ts
Normal file
@@ -0,0 +1,337 @@
|
||||
// Import a Revolut statement (all-time CSV, Excel or PDF export) into one tracker account,
|
||||
// deduped against what's already there (bank-sync rows, Daisy's earlier imports, hand entries).
|
||||
//
|
||||
// bun --env-file=$HOME/.config/money-sync.env scripts/revolut-import.ts \
|
||||
// --file statement.csv --account Joint [options]
|
||||
// (default) dry run: report new rows, duplicates, conflicts, resulting balances
|
||||
// --product Current Revolut sub-account to import (Savings/pockets are reported, skipped)
|
||||
// --currency EUR only this currency (other currency pockets are reported, skipped)
|
||||
// --opening <euros> PDF only: the statement's opening balance (default 0)
|
||||
// --list print every new row / conflict / tracker-only row
|
||||
// --insert-conflicts also insert rows that fall in the already-covered period but match nothing
|
||||
// --insert-conflicts-until YYYY-MM-DD same, but only conflicts up to that date (e.g. a row
|
||||
// the old "Start balance" had absorbed on its own date)
|
||||
// --transfer-pattern <regex> statement rows of type Transfer whose description matches get
|
||||
// category 'transfer' (e.g. Personal <-> Joint legs)
|
||||
// --tag-matched-transfers with --transfer-pattern: also set category 'transfer' on the
|
||||
// existing tracker rows those statement rows matched
|
||||
// --apply write (needs a money-schema backup in ~/backups from the last 2 hours)
|
||||
// --drop-start-balance with --apply: delete the account's "Start balance" row(s), only if the
|
||||
// statement opens at €0 and, after the import, the only difference
|
||||
// to the synced bank balance is the tracker-only rows (hand entries
|
||||
// the bank doesn't have). The sync's balance-adjustment row is then
|
||||
// set to that difference (removed when €0), so the total stays equal
|
||||
// to the bank balance.
|
||||
//
|
||||
// Matching: same amount, date within ±2 days (booking or started date), closest first,
|
||||
// description overlap as tie-breaker. Inserted rows get source 'csv' and a stable
|
||||
// external_id 'revolut-stmt:<hash>', so re-running never duplicates.
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readFileSync, readdirSync, statSync } from 'node:fs';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { homedir } from 'node:os';
|
||||
import postgres from 'postgres';
|
||||
import {
|
||||
checkBalances,
|
||||
parseRevolutCsv,
|
||||
parseRevolutPdfLines,
|
||||
planImport,
|
||||
rowsFromTable,
|
||||
selectRows,
|
||||
statementExternalId,
|
||||
type StatementRow,
|
||||
type TrackerRow
|
||||
} from '../src/lib/server/revolut-statement';
|
||||
import { BALANCE_ADJUSTMENT_ID, isOpeningBalance } from '../src/lib/server/bank-sync';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const value = (name: string) => {
|
||||
const i = args.indexOf(name);
|
||||
return i >= 0 ? args[i + 1] : undefined;
|
||||
};
|
||||
const flag = (name: string) => args.includes(name);
|
||||
const file = value('--file');
|
||||
const accountName = value('--account');
|
||||
if (!file || !accountName)
|
||||
throw new Error('usage: --file <statement> --account <tracker account name>');
|
||||
if (!process.env.DB_URL) throw new Error('DB_URL is not set');
|
||||
const apply = flag('--apply');
|
||||
const euro = (c: number | null | undefined) => (c == null ? '—' : `€${(c / 100).toFixed(2)}`);
|
||||
|
||||
function xlsxTable(path: string): string[][] {
|
||||
const read = (entry: string) => {
|
||||
try {
|
||||
return execFileSync('unzip', ['-p', path, entry], { maxBuffer: 1 << 28 }).toString('utf8');
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
};
|
||||
const unxml = (s: string) =>
|
||||
s
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&');
|
||||
const shared = [...read('xl/sharedStrings.xml').matchAll(/<si>([\s\S]*?)<\/si>/g)].map((m) =>
|
||||
unxml([...m[1].matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((t) => t[1]).join(''))
|
||||
);
|
||||
const sheet = read('xl/worksheets/sheet1.xml');
|
||||
const colIndex = (ref: string) =>
|
||||
[...ref.replace(/\d+/g, '')].reduce((n, ch) => n * 26 + ch.charCodeAt(0) - 64, 0) - 1;
|
||||
return [...sheet.matchAll(/<row[^>]*>([\s\S]*?)<\/row>/g)].map((row) => {
|
||||
const cells: string[] = [];
|
||||
for (const c of row[1].matchAll(/<c r="([A-Z]+\d+)"([^>]*?)(?:\/>|>([\s\S]*?)<\/c>)/g)) {
|
||||
const type = c[2].match(/t="(\w+)"/)?.[1];
|
||||
const v =
|
||||
c[3]?.match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? c[3]?.match(/<t[^>]*>([\s\S]*?)<\/t>/)?.[1] ?? '';
|
||||
cells[colIndex(c[1])] = type === 's' ? (shared[Number(v)] ?? '') : unxml(v);
|
||||
}
|
||||
return Array.from(cells, (x) => x ?? '');
|
||||
});
|
||||
}
|
||||
|
||||
async function pdfLines(path: string): Promise<string[]> {
|
||||
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
|
||||
const doc = await pdfjs.getDocument({ data: new Uint8Array(readFileSync(path)) }).promise;
|
||||
const lines: string[] = [];
|
||||
for (let p = 1; p <= doc.numPages; p++) {
|
||||
const content = await (await doc.getPage(p)).getTextContent();
|
||||
const byY = new Map<number, { x: number; s: string }[]>();
|
||||
for (const item of content.items as { str: string; transform: number[] }[]) {
|
||||
const y = Math.round(item.transform[5]);
|
||||
byY.set(y, [...(byY.get(y) ?? []), { x: item.transform[4], s: item.str }]);
|
||||
}
|
||||
for (const y of [...byY.keys()].sort((a, b) => b - a))
|
||||
lines.push(
|
||||
byY
|
||||
.get(y)!
|
||||
.sort((a, b) => a.x - b.x)
|
||||
.map((i) => i.s)
|
||||
.join(' ')
|
||||
);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
function loadStatement(path: string): StatementRow[] {
|
||||
const lower = path.toLowerCase();
|
||||
if (lower.endsWith('.xlsx')) return rowsFromTable(xlsxTable(path));
|
||||
if (lower.endsWith('.csv') || lower.endsWith('.txt'))
|
||||
return parseRevolutCsv(readFileSync(path, 'utf8'));
|
||||
throw new Error('Use .csv, .xlsx or .pdf');
|
||||
}
|
||||
|
||||
const all = file.toLowerCase().endsWith('.pdf')
|
||||
? parseRevolutPdfLines(await pdfLines(file), Math.round(Number(value('--opening') ?? 0) * 100))
|
||||
: loadStatement(file);
|
||||
const { kept, skipped } = selectRows(
|
||||
all,
|
||||
value('--product') ?? 'Current',
|
||||
value('--currency') ?? 'EUR'
|
||||
);
|
||||
// Chronological by booking date; file order breaks ties (it follows the running balance).
|
||||
const rows = kept.slice().sort((a, b) => a.date.localeCompare(b.date) || a.line - b.line);
|
||||
const balances = checkBalances(kept);
|
||||
|
||||
const sql = postgres(process.env.DB_URL, { max: 2, connect_timeout: 10, onnotice: () => {} });
|
||||
try {
|
||||
const [account] = await sql<
|
||||
{
|
||||
id: number;
|
||||
user_id: string;
|
||||
name: string;
|
||||
opening_balance_cents: number;
|
||||
bank_balance_cents: number | null;
|
||||
bank_balance_at: Date | null;
|
||||
}[]
|
||||
>`select id, user_id, name, opening_balance_cents, bank_balance_cents, bank_balance_at
|
||||
from money.accounts where lower(name) = lower(${accountName})`;
|
||||
if (!account) throw new Error(`No tracker account named ${accountName}`);
|
||||
const existing = await sql<
|
||||
{
|
||||
id: number;
|
||||
date: string;
|
||||
amount_cents: number;
|
||||
description: string;
|
||||
source: string;
|
||||
external_id: string | null;
|
||||
text: string;
|
||||
}[]
|
||||
>`select id, date::text, amount_cents, description, source, external_id,
|
||||
concat_ws(' ', description, counterparty, raw->'creditor'->>'name', raw->'debtor'->>'name',
|
||||
raw->>'remittance_information', raw->'revolut'->>'description') as text
|
||||
from money.transactions where account_id = ${account.id} order by date, id`;
|
||||
const startRows = existing.filter(
|
||||
(t) => isOpeningBalance(t.description) && t.source === 'manual'
|
||||
);
|
||||
const adjustment = existing.find((t) => t.external_id === BALANCE_ADJUSTMENT_ID);
|
||||
const tracker: TrackerRow[] = existing
|
||||
.filter((t) => !startRows.includes(t) && t !== adjustment)
|
||||
.map((t) => ({
|
||||
id: t.id,
|
||||
date: t.date,
|
||||
amountCents: t.amount_cents,
|
||||
description: t.description,
|
||||
text: t.text,
|
||||
source: t.source,
|
||||
externalId: t.external_id
|
||||
}));
|
||||
const plan = planImport(rows, tracker);
|
||||
const conflictsUntil = value('--insert-conflicts-until');
|
||||
const toInsert = flag('--insert-conflicts')
|
||||
? [...plan.fresh, ...plan.conflicts]
|
||||
: conflictsUntil
|
||||
? [...plan.fresh, ...plan.conflicts.filter((r) => r.date <= conflictsUntil)]
|
||||
: plan.fresh;
|
||||
const transferRe = value('--transfer-pattern')
|
||||
? new RegExp(value('--transfer-pattern')!, 'i')
|
||||
: null;
|
||||
const isTransfer = (r: StatementRow) =>
|
||||
!!transferRe && /transfer/i.test(r.type) && transferRe.test(r.description);
|
||||
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
const sum = (xs: { amountCents: number; date: string }[]) =>
|
||||
xs.filter((x) => x.date <= today).reduce((s, x) => s + x.amountCents, 0);
|
||||
const trackedNow =
|
||||
account.opening_balance_cents +
|
||||
sum(tracker) +
|
||||
sum(startRows.map((t) => ({ amountCents: t.amount_cents, date: t.date }))) +
|
||||
(adjustment?.amount_cents ?? 0);
|
||||
const afterNoStart = account.opening_balance_cents + sum(tracker) + sum(toInsert);
|
||||
const bank = account.bank_balance_cents;
|
||||
const residual = bank == null ? null : bank - afterNoStart;
|
||||
const trackerOnlyCents = sum(plan.trackerOnly);
|
||||
const dropOk = balances.openingCents === 0 && residual !== null && residual === -trackerOnlyCents;
|
||||
const boundary = plan.coverageFrom
|
||||
? (rows.filter((r) => r.date < plan.coverageFrom! && r.balanceCents != null).at(-1)
|
||||
?.balanceCents ?? 0)
|
||||
: null;
|
||||
|
||||
console.log(
|
||||
`Statement ${file}: ${all.length} rows; importing ${account.name} = ${value('--product') ?? 'Current'} ${value('--currency') ?? 'EUR'}: ${kept.length} completed rows ${rows[0]?.date ?? ''}..${rows.at(-1)?.date ?? ''}`
|
||||
);
|
||||
const skippedBy = new Map<string, { n: number; cents: number }>();
|
||||
for (const r of skipped) {
|
||||
const key = `${r.product}/${r.currency}/${r.state}`;
|
||||
const e = skippedBy.get(key) ?? { n: 0, cents: 0 };
|
||||
skippedBy.set(key, { n: e.n + 1, cents: e.cents + r.amountCents });
|
||||
}
|
||||
for (const [k, v] of skippedBy) console.log(` skipped ${k}: ${v.n} rows (net ${euro(v.cents)})`);
|
||||
console.log(
|
||||
` statement opening ${euro(balances.openingCents)}, closing ${euro(balances.closingCents)}, running-balance breaks ${balances.breaks.length}`
|
||||
);
|
||||
console.log(
|
||||
`Tracker ${account.name} (#${account.id}): ${tracker.length} rows from ${plan.coverageFrom ?? '-'}; start balance rows ${startRows.map((t) => `#${t.id} ${t.date} ${euro(t.amount_cents)}`).join(', ') || 'none'}; adjustment ${euro(adjustment?.amount_cents)}`
|
||||
);
|
||||
console.log(
|
||||
` statement balance just before ${plan.coverageFrom}: ${euro(boundary)} (tracker start + adjustment: ${euro(sum(startRows.map((t) => ({ amountCents: t.amount_cents, date: t.date }))) + (adjustment?.amount_cents ?? 0))})`
|
||||
);
|
||||
console.log(
|
||||
`Plan: new ${plan.fresh.length} (${euro(sum(plan.fresh))}), duplicates ${plan.duplicates.length} (${plan.duplicates.filter((d) => !d.descMatch).length} with differing description), conflicts ${plan.conflicts.length} (${euro(sum(plan.conflicts))}), tracker-only ${plan.trackerOnly.length} (${euro(sum(plan.trackerOnly))})`
|
||||
);
|
||||
console.log(
|
||||
`Balances: bank ${euro(bank)} (synced ${account.bank_balance_at?.toISOString() ?? 'never'}), tracker now ${euro(trackedNow)}, after import without start balance/adjustment ${euro(afterNoStart)}, gap ${euro(residual)}${residual !== null && residual === -trackerOnlyCents ? ' = tracker-only rows, OK to drop the start balance' : ` (tracker-only rows ${euro(trackerOnlyCents)}; start balance must stay)`}${transferRe ? `; transfers matching pattern: ${rows.filter(isTransfer).length}` : ''}`
|
||||
);
|
||||
if (flag('--list') || !apply) {
|
||||
const show = (
|
||||
label: string,
|
||||
xs: {
|
||||
date: string;
|
||||
amountCents: number;
|
||||
description: string;
|
||||
id?: number;
|
||||
source?: string;
|
||||
}[],
|
||||
limit: number
|
||||
) => {
|
||||
if (!xs.length) return;
|
||||
console.log(` ${label}:`);
|
||||
for (const x of xs.slice(0, flag('--list') ? Infinity : limit))
|
||||
console.log(
|
||||
` ${x.date} ${euro(x.amountCents).padStart(10)} ${x.description}${x.id ? ` (#${x.id} ${x.source})` : ''}`
|
||||
);
|
||||
if (!flag('--list') && xs.length > limit)
|
||||
console.log(` … ${xs.length - limit} more (--list)`);
|
||||
};
|
||||
show('conflicts (statement only, in covered period)', plan.conflicts, 30);
|
||||
show('tracker-only rows (covered period, no statement row)', plan.trackerOnly, 30);
|
||||
show(
|
||||
'duplicates with differing description',
|
||||
plan.duplicates
|
||||
.filter((d) => !d.descMatch)
|
||||
.map((d) => ({
|
||||
...d.row,
|
||||
description: `${d.row.description} == ${d.match.description}`
|
||||
})),
|
||||
10
|
||||
);
|
||||
show('new rows', plan.fresh, 5);
|
||||
}
|
||||
if (!apply) {
|
||||
console.log('Dry run: nothing written. Re-run with --apply (after a backup) to import.');
|
||||
} else {
|
||||
const dir = `${homedir()}/backups`;
|
||||
const fresh = readdirSync(dir).some(
|
||||
(f) =>
|
||||
f.startsWith('money-before-') &&
|
||||
f.endsWith('.dump') &&
|
||||
Date.now() - statSync(`${dir}/${f}`).mtimeMs < 2 * 3600_000
|
||||
);
|
||||
if (!fresh)
|
||||
throw new Error(`No money-before-*.dump in ${dir} from the last 2 hours; back up first`);
|
||||
const drop = flag('--drop-start-balance');
|
||||
if (drop && !dropOk)
|
||||
throw new Error(
|
||||
`Refusing --drop-start-balance: statement opening ${euro(balances.openingCents)}, gap ${euro(residual)} vs tracker-only rows ${euro(trackerOnlyCents)}`
|
||||
);
|
||||
const hash = (s: string) => createHash('sha256').update(s).digest('hex');
|
||||
await sql.begin(async (q) => {
|
||||
let inserted = 0;
|
||||
for (const r of toInsert) {
|
||||
const res = await q`
|
||||
insert into money.transactions
|
||||
(user_id, account_id, date, value_date, description, amount_cents, external_id, source, booking_status, category, raw)
|
||||
values (${account.user_id}, ${account.id}, ${r.date}, ${r.startedDate}, ${r.description.slice(0, 500)},
|
||||
${r.amountCents}, ${statementExternalId(r, hash)}, 'csv', 'BOOK', ${isTransfer(r) ? 'transfer' : null},
|
||||
${q.json({ revolut: { type: r.type, product: r.product, started: r.startedDate, completed: r.completedAt, description: r.description, fee_cents: r.feeCents, currency: r.currency, balance_cents: r.balanceCents, file_line: r.line } })})
|
||||
on conflict do nothing`;
|
||||
inserted += res.count;
|
||||
}
|
||||
console.log(`Inserted ${inserted} rows into ${account.name}.`);
|
||||
if (transferRe && flag('--tag-matched-transfers')) {
|
||||
const ids = plan.duplicates.filter((d) => isTransfer(d.row)).map((d) => d.match.id);
|
||||
const tagged = ids.length
|
||||
? await q`update money.transactions set category = 'transfer'
|
||||
where account_id = ${account.id} and id in ${q(ids)} and category is distinct from 'transfer'`
|
||||
: { count: 0 };
|
||||
console.log(`Tagged ${tagged.count} existing transfer rows.`);
|
||||
}
|
||||
if (drop) {
|
||||
for (const t of startRows)
|
||||
await q`delete from money.transactions where id = ${t.id} and account_id = ${account.id}`;
|
||||
const needed = residual!;
|
||||
if (adjustment && needed === 0)
|
||||
await q`delete from money.transactions where id = ${adjustment.id}`;
|
||||
else if (adjustment)
|
||||
await q`update money.transactions set amount_cents = ${needed} where id = ${adjustment.id}`;
|
||||
else if (needed !== 0)
|
||||
await q`insert into money.transactions
|
||||
(user_id, account_id, date, description, amount_cents, external_id, source, booking_status)
|
||||
values (${account.user_id}, ${account.id}, ${plan.coverageFrom ?? rows[0].date}, 'Bank balance adjustment',
|
||||
${needed}, ${BALANCE_ADJUSTMENT_ID}, 'enablebanking', 'ADJ')`;
|
||||
const [check] = await q<{ total: number }[]>`
|
||||
select (a.opening_balance_cents + coalesce(sum(t.amount_cents) filter (where t.date <= ${today}), 0))::int as total
|
||||
from money.accounts a left join money.transactions t on t.account_id = a.id
|
||||
where a.id = ${account.id} group by a.id`;
|
||||
if (bank != null && check.total !== bank)
|
||||
throw new Error(`Total would be ${euro(check.total)}, bank ${euro(bank)}; rolled back`);
|
||||
console.log(
|
||||
`Removed start balance ${startRows.map((t) => '#' + t.id).join(', ') || '-'}; adjustment ${euro(adjustment?.amount_cents)} -> ${euro(needed)}; total ${euro(check.total)} = bank.`
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
103
src/lib/server/revolut-statement.test.ts
Normal file
103
src/lib/server/revolut-statement.test.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
checkBalances,
|
||||
parseAmount,
|
||||
parseRevolutCsv,
|
||||
parseRevolutPdfLines,
|
||||
planImport,
|
||||
selectRows,
|
||||
type TrackerRow
|
||||
} from './revolut-statement';
|
||||
|
||||
const CSV = `Type,Product,Started Date,Completed Date,Description,Amount,Fee,Currency,State,Balance
|
||||
Transfer,Current,2024-06-30 15:09:04,2024-06-30 15:09:04,Transfer from STAN RUNGE,100.00,0.00,EUR,COMPLETED,100.00
|
||||
Card Payment,Current,2024-07-01 19:12:32,2024-07-02 13:57:17,"Wok, Baldakijn",-72.10,0.00,EUR,COMPLETED,27.90
|
||||
Card Payment,Current,2024-07-03 10:00:00,,Shop,-5.00,0.00,EUR,REVERTED,
|
||||
Exchange,Current,2024-07-04 10:00:00,2024-07-04 10:00:01,Exchanged to USD,-10.00,0.20,EUR,COMPLETED,17.70
|
||||
Exchange,Current,2024-07-04 10:00:00,2024-07-04 10:00:01,Exchanged from EUR,10.80,0.00,USD,COMPLETED,10.80
|
||||
CASHBACK,Savings,2024-08-09 18:56:54,2024-08-10 08:42:12,Plan Cashback,0.01,0.00,EUR,COMPLETED,0.01
|
||||
Card Payment,Current,2025-08-02 10:00:00,2025-08-03 09:00:00,Albert Heijn 1234,-12.34,0.00,EUR,COMPLETED,5.36
|
||||
Card Payment,Current,2025-08-05 10:00:00,2025-08-05 11:00:00,Kiosk,-3.50,0.00,EUR,COMPLETED,1.86
|
||||
`;
|
||||
|
||||
describe('revolut statement', () => {
|
||||
it('parses amounts in both notations', () => {
|
||||
expect(parseAmount('1,234.56')).toBe(123456);
|
||||
expect(parseAmount('-1.234,56')).toBe(-123456);
|
||||
expect(parseAmount('€ 12,50')).toBe(1250);
|
||||
expect(parseAmount('')).toBeNull();
|
||||
});
|
||||
|
||||
it('parses the CSV export, nets fees and keeps only completed Current EUR rows', () => {
|
||||
const rows = parseRevolutCsv(CSV);
|
||||
expect(rows).toHaveLength(8);
|
||||
expect(rows[1]).toMatchObject({
|
||||
date: '2024-07-02',
|
||||
startedDate: '2024-07-01',
|
||||
description: 'Wok, Baldakijn',
|
||||
amountCents: -7210
|
||||
});
|
||||
const { kept, skipped } = selectRows(rows);
|
||||
expect(kept.map((r) => r.amountCents)).toEqual([10000, -7210, -1020, -1234, -350]);
|
||||
expect(skipped.map((r) => `${r.product}/${r.currency}/${r.state}`)).toEqual([
|
||||
'Current/EUR/REVERTED',
|
||||
'Current/USD/COMPLETED',
|
||||
'Savings/EUR/COMPLETED'
|
||||
]);
|
||||
expect(checkBalances(kept)).toMatchObject({ openingCents: 0, closingCents: 186, breaks: [] });
|
||||
});
|
||||
|
||||
it('reads Dutch semicolon headers', () => {
|
||||
const rows = parseRevolutCsv(
|
||||
'Soort;Product;Begindatum;Datum voltooid;Beschrijving;Bedrag;Kosten;Valuta;Status;Saldo\nKaartbetaling;Current;2024-01-02 10:00:00;2024-01-03 10:00:00;Jumbo;-3,50;0,00;EUR;VOLTOOID;10,00\n'
|
||||
);
|
||||
expect(rows[0]).toMatchObject({ date: '2024-01-03', amountCents: -350, balanceCents: 1000 });
|
||||
});
|
||||
|
||||
it('parses PDF statement lines using the running balance for direction', () => {
|
||||
const rows = parseRevolutPdfLines([
|
||||
'EUR Statement',
|
||||
'30 Jun 2024 Transfer from STAN RUNGE €100.00 €100.00',
|
||||
'Jul 2, 2024 Wok Van Baldakijn €72.10 €27.90',
|
||||
'2 jul. 2024 Something €1.234,00 €1.261,90',
|
||||
'Balance summary €0.00 €27.90'
|
||||
]);
|
||||
expect(rows.map((r) => [r.date, r.amountCents])).toEqual([
|
||||
['2024-06-30', 10000],
|
||||
['2024-07-02', -7210],
|
||||
['2024-07-02', 123400]
|
||||
]);
|
||||
});
|
||||
|
||||
it('plans an import: duplicates within ±2 days, older unmatched rows are new, newer are conflicts', () => {
|
||||
const { kept } = selectRows(parseRevolutCsv(CSV));
|
||||
const tracker: TrackerRow[] = [
|
||||
{
|
||||
id: 1,
|
||||
date: '2025-08-02',
|
||||
amountCents: -1234,
|
||||
description: 'Groceries',
|
||||
text: 'Groceries ALBERT HEIJN 1234',
|
||||
source: 'manual',
|
||||
externalId: 'x'
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
date: '2025-08-04',
|
||||
amountCents: -999,
|
||||
description: 'Cash',
|
||||
text: 'Cash',
|
||||
source: 'manual',
|
||||
externalId: null
|
||||
}
|
||||
];
|
||||
const plan = planImport(kept, tracker);
|
||||
expect(plan.coverageFrom).toBe('2025-08-02');
|
||||
expect(plan.duplicates.map((d) => [d.row.amountCents, d.match.id, d.descMatch])).toEqual([
|
||||
[-1234, 1, true]
|
||||
]);
|
||||
expect(plan.fresh.map((r) => r.amountCents)).toEqual([10000, -7210, -1020]);
|
||||
expect(plan.conflicts.map((r) => r.amountCents)).toEqual([-350]);
|
||||
expect(plan.trackerOnly.map((t) => t.id)).toEqual([2]);
|
||||
});
|
||||
});
|
||||
343
src/lib/server/revolut-statement.ts
Normal file
343
src/lib/server/revolut-statement.ts
Normal file
@@ -0,0 +1,343 @@
|
||||
// Revolut account statement parsing + import planning (pure; DB work is in scripts/revolut-import.ts).
|
||||
// Handles the app's CSV export (English or Dutch headers, , or ; separated), the Excel export
|
||||
// (rows passed in as a string table) and, defensively, the text of the PDF statement.
|
||||
|
||||
export type StatementRow = {
|
||||
/** Booking date (completed date, else started date), YYYY-MM-DD. */
|
||||
date: string;
|
||||
startedDate: string;
|
||||
completedAt: string;
|
||||
type: string;
|
||||
product: string;
|
||||
description: string;
|
||||
/** Net balance change: amount minus fee. */
|
||||
amountCents: number;
|
||||
feeCents: number;
|
||||
currency: string;
|
||||
state: string;
|
||||
balanceCents: number | null;
|
||||
/** Position in the file, for stable ordering. */
|
||||
line: number;
|
||||
};
|
||||
|
||||
const HEADERS: Record<string, string[]> = {
|
||||
type: ['type', 'soort'],
|
||||
product: ['product'],
|
||||
started: ['started date', 'begindatum', 'startdatum', 'datum gestart'],
|
||||
completed: ['completed date', 'datum voltooid', 'voltooid', 'einddatum'],
|
||||
description: ['description', 'beschrijving', 'omschrijving'],
|
||||
amount: ['amount', 'bedrag'],
|
||||
fee: ['fee', 'kosten', 'vergoeding'],
|
||||
currency: ['currency', 'valuta'],
|
||||
state: ['state', 'status'],
|
||||
balance: ['balance', 'saldo']
|
||||
};
|
||||
|
||||
/** Parses "1,234.56", "-1.234,56", "€ 12,50", "1234" into cents. */
|
||||
export function parseAmount(raw: string | undefined | null): number | null {
|
||||
if (raw == null) return null;
|
||||
let s = String(raw)
|
||||
.replace(/[€$£\s\u00a0]|EUR|USD/g, '')
|
||||
.replace(/^\((.*)\)$/, '-$1')
|
||||
.replace(/[−–]/g, '-');
|
||||
if (!s || s === '-') return null;
|
||||
const lastComma = s.lastIndexOf(',');
|
||||
const lastDot = s.lastIndexOf('.');
|
||||
if (lastComma > lastDot) s = s.replace(/\./g, '').replace(',', '.');
|
||||
else s = s.replace(/,/g, '');
|
||||
if (!/^[-+]?\d+(\.\d+)?$/.test(s)) return null;
|
||||
return Math.round(Number(s) * 100);
|
||||
}
|
||||
|
||||
/** RFC 4180-ish CSV parser; detects , or ; from the header line. */
|
||||
export function parseCsv(text: string): string[][] {
|
||||
text = text.replace(/^\uFEFF/, '');
|
||||
const firstLine = text.slice(0, text.indexOf('\n') >= 0 ? text.indexOf('\n') : text.length);
|
||||
const sep =
|
||||
(firstLine.match(/;/g)?.length ?? 0) > (firstLine.match(/,/g)?.length ?? 0) ? ';' : ',';
|
||||
const rows: string[][] = [];
|
||||
let row: string[] = [];
|
||||
let field = '';
|
||||
let quoted = false;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const c = text[i];
|
||||
if (quoted) {
|
||||
if (c === '"' && text[i + 1] === '"') {
|
||||
field += '"';
|
||||
i++;
|
||||
} else if (c === '"') quoted = false;
|
||||
else field += c;
|
||||
} else if (c === '"') quoted = true;
|
||||
else if (c === sep) {
|
||||
row.push(field);
|
||||
field = '';
|
||||
} else if (c === '\n' || c === '\r') {
|
||||
if (c === '\r' && text[i + 1] === '\n') i++;
|
||||
row.push(field);
|
||||
if (row.some((f) => f !== '')) rows.push(row);
|
||||
row = [];
|
||||
field = '';
|
||||
} else field += c;
|
||||
}
|
||||
row.push(field);
|
||||
if (row.some((f) => f !== '')) rows.push(row);
|
||||
return rows;
|
||||
}
|
||||
|
||||
function isoDay(value: string): string {
|
||||
const v = value.trim();
|
||||
let m = v.match(/^(\d{4})-(\d{2})-(\d{2})/);
|
||||
if (m) return `${m[1]}-${m[2]}-${m[3]}`;
|
||||
m = v.match(/^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})/);
|
||||
if (m) return `${m[3]}-${m[2].padStart(2, '0')}-${m[1].padStart(2, '0')}`;
|
||||
// Excel serial day number
|
||||
if (/^\d{5}(\.\d+)?$/.test(v)) {
|
||||
const d = new Date(Math.round((Number(v) - 25569) * 86400) * 1000);
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
/** Turns a header + rows table (CSV or Excel) into statement rows. */
|
||||
export function rowsFromTable(table: string[][]): StatementRow[] {
|
||||
const header = (table[0] ?? []).map((h) => h.trim().toLowerCase());
|
||||
const col = (key: string) => header.findIndex((h) => HEADERS[key].includes(h));
|
||||
const idx = Object.fromEntries(Object.keys(HEADERS).map((k) => [k, col(k)]));
|
||||
for (const key of ['started', 'description', 'amount'])
|
||||
if (idx[key] < 0) throw new Error(`Not a Revolut statement: no "${HEADERS[key][0]}" column`);
|
||||
const get = (r: string[], key: string) => (idx[key] >= 0 ? (r[idx[key]] ?? '').trim() : '');
|
||||
return table.slice(1).flatMap((r, i) => {
|
||||
const amount = parseAmount(get(r, 'amount'));
|
||||
const started = isoDay(get(r, 'started'));
|
||||
if (amount == null || !started) return [];
|
||||
const fee = parseAmount(get(r, 'fee')) ?? 0;
|
||||
const completedAt = get(r, 'completed');
|
||||
return [
|
||||
{
|
||||
date: isoDay(completedAt) || started,
|
||||
startedDate: started,
|
||||
completedAt,
|
||||
type: get(r, 'type'),
|
||||
product: get(r, 'product') || 'Current',
|
||||
description: get(r, 'description'),
|
||||
amountCents: amount - fee,
|
||||
feeCents: fee,
|
||||
currency: (get(r, 'currency') || 'EUR').toUpperCase(),
|
||||
state: (get(r, 'state') || 'COMPLETED').toUpperCase(),
|
||||
balanceCents: parseAmount(get(r, 'balance')),
|
||||
line: i + 2
|
||||
}
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
export function parseRevolutCsv(text: string): StatementRow[] {
|
||||
return rowsFromTable(parseCsv(text));
|
||||
}
|
||||
|
||||
const MONTHS: Record<string, number> = {
|
||||
jan: 1,
|
||||
feb: 2,
|
||||
mar: 3,
|
||||
mrt: 3,
|
||||
apr: 4,
|
||||
may: 5,
|
||||
mei: 5,
|
||||
jun: 6,
|
||||
jul: 7,
|
||||
aug: 8,
|
||||
sep: 9,
|
||||
oct: 10,
|
||||
okt: 10,
|
||||
nov: 11,
|
||||
dec: 12
|
||||
};
|
||||
const PDF_DATE =
|
||||
/^(?:(\d{1,2})\s+([a-z]{3})[a-z]*\.?\s+(\d{4})|([a-z]{3})[a-z]*\.?\s+(\d{1,2}),?\s+(\d{4}))\s+/i;
|
||||
const MONEY = /[-−]?€\s?[\d.,]+|[-−]?[\d.,]+\s?€/g;
|
||||
|
||||
/**
|
||||
* Best-effort parser for the text lines of a Revolut PDF statement ("Date Description
|
||||
* Money out Money in Balance"). In/out is taken from the balance change, so every
|
||||
* transaction line must end in its running balance; `openingCents` is the statement's
|
||||
* opening balance (0 for an all-time statement).
|
||||
*/
|
||||
export function parseRevolutPdfLines(lines: string[], openingCents = 0): StatementRow[] {
|
||||
const out: StatementRow[] = [];
|
||||
let balance = openingCents;
|
||||
lines.forEach((raw, i) => {
|
||||
const line = raw.replace(/\s+/g, ' ').trim();
|
||||
const m = line.match(PDF_DATE);
|
||||
if (!m) return;
|
||||
const day = Number(m[1] ?? m[5]);
|
||||
const month = MONTHS[(m[2] ?? m[4]).toLowerCase()];
|
||||
const year = Number(m[3] ?? m[6]);
|
||||
if (!month || !day) return;
|
||||
const rest = line.slice(m[0].length);
|
||||
const money = rest.match(MONEY);
|
||||
if (!money || money.length < 2) return;
|
||||
const next = parseAmount(money.at(-1)!.replace('−', '-'));
|
||||
const amount = parseAmount(money.at(-2)!.replace('−', '-'));
|
||||
if (next == null || amount == null) return;
|
||||
const description = rest.slice(0, rest.indexOf(money[0])).trim();
|
||||
const delta = next - balance;
|
||||
if (Math.abs(delta) !== Math.abs(amount)) return; // not a transaction line (or out of order)
|
||||
const date = `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
|
||||
out.push({
|
||||
date,
|
||||
startedDate: date,
|
||||
completedAt: date,
|
||||
type: '',
|
||||
product: 'Current',
|
||||
description,
|
||||
amountCents: delta,
|
||||
feeCents: 0,
|
||||
currency: 'EUR',
|
||||
state: 'COMPLETED',
|
||||
balanceCents: next,
|
||||
line: i + 1
|
||||
});
|
||||
balance = next;
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Rows that move money in the selected sub-account (completed only, one product + currency). */
|
||||
export function selectRows(rows: StatementRow[], product = 'Current', currency = 'EUR') {
|
||||
const kept = rows.filter(
|
||||
(r) =>
|
||||
r.state === 'COMPLETED' &&
|
||||
r.product.toLowerCase() === product.toLowerCase() &&
|
||||
r.currency === currency
|
||||
);
|
||||
const skipped = rows.filter((r) => !kept.includes(r));
|
||||
return { kept, skipped };
|
||||
}
|
||||
|
||||
/** Checks the running balance column; returns the balance before the first row. */
|
||||
export function checkBalances(rows: StatementRow[]) {
|
||||
const breaks: StatementRow[] = [];
|
||||
let prev: number | null = null;
|
||||
for (const r of rows) {
|
||||
if (r.balanceCents == null) continue;
|
||||
if (prev != null && prev + r.amountCents !== r.balanceCents) breaks.push(r);
|
||||
prev = r.balanceCents;
|
||||
}
|
||||
const first = rows.find((r) => r.balanceCents != null);
|
||||
return {
|
||||
openingCents: first ? first.balanceCents! - first.amountCents : null,
|
||||
closingCents: prev,
|
||||
breaks
|
||||
};
|
||||
}
|
||||
|
||||
export type TrackerRow = {
|
||||
id: number;
|
||||
date: string;
|
||||
amountCents: number;
|
||||
description: string;
|
||||
/** Description, counterparty and raw bank names/remittance, for matching. */
|
||||
text: string;
|
||||
source: string;
|
||||
externalId: string | null;
|
||||
};
|
||||
|
||||
const STOP = new Set([
|
||||
'transfer',
|
||||
'from',
|
||||
'to',
|
||||
'payment',
|
||||
'the',
|
||||
'and',
|
||||
'van',
|
||||
'naar',
|
||||
'aan',
|
||||
'bv',
|
||||
'b.v',
|
||||
'eur'
|
||||
]);
|
||||
export function words(text: string) {
|
||||
return new Set(
|
||||
text
|
||||
.toLowerCase()
|
||||
.normalize('NFKD')
|
||||
.replace(/[^a-z0-9 ]+/g, ' ')
|
||||
.split(/\s+/)
|
||||
.filter((w) => w.length > 2 && !STOP.has(w))
|
||||
);
|
||||
}
|
||||
|
||||
function days(a: string, b: string) {
|
||||
return Math.abs(Date.parse(a + 'T00:00:00Z') - Date.parse(b + 'T00:00:00Z')) / 86_400_000;
|
||||
}
|
||||
|
||||
export type ImportPlan = {
|
||||
coverageFrom: string | null;
|
||||
fresh: StatementRow[];
|
||||
duplicates: { row: StatementRow; match: TrackerRow; descMatch: boolean }[];
|
||||
/** In the period the tracker already covers, but no tracker row matches. */
|
||||
conflicts: StatementRow[];
|
||||
/** Tracker rows in the covered period that no statement row matched. */
|
||||
trackerOnly: TrackerRow[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Matches statement rows to tracker rows: same amount, booking or started date within
|
||||
* `maxDays`, closest date first, description overlap breaking ties. Unmatched statement
|
||||
* rows before the tracker's first real row are new; unmatched ones after it are conflicts.
|
||||
*/
|
||||
export function planImport(
|
||||
rows: StatementRow[],
|
||||
tracker: TrackerRow[],
|
||||
{ maxDays = 2, coverageFrom }: { maxDays?: number; coverageFrom?: string | null } = {}
|
||||
): ImportPlan {
|
||||
const from = coverageFrom ?? tracker.map((t) => t.date).sort()[0] ?? null;
|
||||
const trackerWords = new Map(tracker.map((t) => [t.id, words(t.text)]));
|
||||
const pairs: { r: number; t: TrackerRow; score: number; desc: boolean }[] = [];
|
||||
const byAmount = new Map<number, TrackerRow[]>();
|
||||
for (const t of tracker) byAmount.set(t.amountCents, [...(byAmount.get(t.amountCents) ?? []), t]);
|
||||
rows.forEach((row, r) => {
|
||||
const w = words(row.description);
|
||||
for (const t of byAmount.get(row.amountCents) ?? []) {
|
||||
const diff = Math.min(days(row.date, t.date), days(row.startedDate, t.date));
|
||||
if (diff > maxDays) continue;
|
||||
const tw = trackerWords.get(t.id)!;
|
||||
const desc = [...w].some((x) => tw.has(x));
|
||||
pairs.push({ r, t, score: diff * 10 - (desc ? 5 : 0) + days(row.date, t.date) * 0.1, desc });
|
||||
}
|
||||
});
|
||||
pairs.sort((a, b) => a.score - b.score || a.r - b.r || a.t.id - b.t.id);
|
||||
const usedRows = new Set<number>();
|
||||
const usedTracker = new Set<number>();
|
||||
const duplicates: ImportPlan['duplicates'] = [];
|
||||
for (const p of pairs) {
|
||||
if (usedRows.has(p.r) || usedTracker.has(p.t.id)) continue;
|
||||
usedRows.add(p.r);
|
||||
usedTracker.add(p.t.id);
|
||||
duplicates.push({ row: rows[p.r], match: p.t, descMatch: p.desc });
|
||||
}
|
||||
const unmatched = rows.filter((_, i) => !usedRows.has(i));
|
||||
return {
|
||||
coverageFrom: from,
|
||||
fresh: unmatched.filter((r) => !from || r.date < from),
|
||||
duplicates,
|
||||
conflicts: unmatched.filter((r) => from && r.date >= from),
|
||||
trackerOnly: tracker.filter((t) => !usedTracker.has(t.id) && (!from || t.date >= from))
|
||||
};
|
||||
}
|
||||
|
||||
/** Stable id so re-running an import never inserts a row twice. */
|
||||
export function statementExternalId(row: StatementRow, hash: (s: string) => string) {
|
||||
return (
|
||||
'revolut-stmt:' +
|
||||
hash(
|
||||
[
|
||||
row.completedAt || row.startedDate,
|
||||
row.startedDate,
|
||||
row.amountCents,
|
||||
row.description,
|
||||
row.balanceCents ?? ''
|
||||
].join('|')
|
||||
).slice(0, 24)
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user