Stan Runge
8dde5aa302
All checks were successful
Deploy production / deploy (push) Successful in 2m34s
338 lines
16 KiB
TypeScript
338 lines
16 KiB
TypeScript
// Import a Revolut statement (all-time CSV, Excel or PDF export) into one tracker account,
|
|
// deduped against what's already there (bank-sync rows, Daisy's earlier imports, hand entries).
|
|
//
|
|
// bun --env-file=$HOME/.config/money-sync.env scripts/revolut-import.ts \
|
|
// --file statement.csv --account Joint [options]
|
|
// (default) dry run: report new rows, duplicates, conflicts, resulting balances
|
|
// --product Current Revolut sub-account to import (Savings/pockets are reported, skipped)
|
|
// --currency EUR only this currency (other currency pockets are reported, skipped)
|
|
// --opening <euros> PDF only: the statement's opening balance (default 0)
|
|
// --list print every new row / conflict / tracker-only row
|
|
// --insert-conflicts also insert rows that fall in the already-covered period but match nothing
|
|
// --insert-conflicts-until YYYY-MM-DD same, but only conflicts up to that date (e.g. a row
|
|
// the old "Start balance" had absorbed on its own date)
|
|
// --transfer-pattern <regex> statement rows of type Transfer whose description matches get
|
|
// category 'transfer' (e.g. Personal <-> Joint legs)
|
|
// --tag-matched-transfers with --transfer-pattern: also set category 'transfer' on the
|
|
// existing tracker rows those statement rows matched
|
|
// --apply write (needs a money-schema backup in ~/backups from the last 2 hours)
|
|
// --drop-start-balance with --apply: delete the account's "Start balance" row(s), only if the
|
|
// statement opens at €0 and, after the import, the only difference
|
|
// to the synced bank balance is the tracker-only rows (hand entries
|
|
// the bank doesn't have). The sync's balance-adjustment row is then
|
|
// set to that difference (removed when €0), so the total stays equal
|
|
// to the bank balance.
|
|
//
|
|
// Matching: same amount, date within ±2 days (booking or started date), closest first,
|
|
// description overlap as tie-breaker. Inserted rows get source 'csv' and a stable
|
|
// external_id 'revolut-stmt:<hash>', so re-running never duplicates.
|
|
import { createHash } from 'node:crypto';
|
|
import { readFileSync, readdirSync, statSync } from 'node:fs';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { homedir } from 'node:os';
|
|
import postgres from 'postgres';
|
|
import {
|
|
checkBalances,
|
|
parseRevolutCsv,
|
|
parseRevolutPdfLines,
|
|
planImport,
|
|
rowsFromTable,
|
|
selectRows,
|
|
statementExternalId,
|
|
type StatementRow,
|
|
type TrackerRow
|
|
} from '../src/lib/server/revolut-statement';
|
|
import { BALANCE_ADJUSTMENT_ID, isOpeningBalance } from '../src/lib/server/bank-sync';
|
|
|
|
const args = process.argv.slice(2);
|
|
const value = (name: string) => {
|
|
const i = args.indexOf(name);
|
|
return i >= 0 ? args[i + 1] : undefined;
|
|
};
|
|
const flag = (name: string) => args.includes(name);
|
|
const file = value('--file');
|
|
const accountName = value('--account');
|
|
if (!file || !accountName)
|
|
throw new Error('usage: --file <statement> --account <tracker account name>');
|
|
if (!process.env.DB_URL) throw new Error('DB_URL is not set');
|
|
const apply = flag('--apply');
|
|
const euro = (c: number | null | undefined) => (c == null ? '—' : `€${(c / 100).toFixed(2)}`);
|
|
|
|
function xlsxTable(path: string): string[][] {
|
|
const read = (entry: string) => {
|
|
try {
|
|
return execFileSync('unzip', ['-p', path, entry], { maxBuffer: 1 << 28 }).toString('utf8');
|
|
} catch {
|
|
return '';
|
|
}
|
|
};
|
|
const unxml = (s: string) =>
|
|
s
|
|
.replace(/</g, '<')
|
|
.replace(/>/g, '>')
|
|
.replace(/"/g, '"')
|
|
.replace(/'/g, "'")
|
|
.replace(/&/g, '&');
|
|
const shared = [...read('xl/sharedStrings.xml').matchAll(/<si>([\s\S]*?)<\/si>/g)].map((m) =>
|
|
unxml([...m[1].matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((t) => t[1]).join(''))
|
|
);
|
|
const sheet = read('xl/worksheets/sheet1.xml');
|
|
const colIndex = (ref: string) =>
|
|
[...ref.replace(/\d+/g, '')].reduce((n, ch) => n * 26 + ch.charCodeAt(0) - 64, 0) - 1;
|
|
return [...sheet.matchAll(/<row[^>]*>([\s\S]*?)<\/row>/g)].map((row) => {
|
|
const cells: string[] = [];
|
|
for (const c of row[1].matchAll(/<c r="([A-Z]+\d+)"([^>]*?)(?:\/>|>([\s\S]*?)<\/c>)/g)) {
|
|
const type = c[2].match(/t="(\w+)"/)?.[1];
|
|
const v =
|
|
c[3]?.match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? c[3]?.match(/<t[^>]*>([\s\S]*?)<\/t>/)?.[1] ?? '';
|
|
cells[colIndex(c[1])] = type === 's' ? (shared[Number(v)] ?? '') : unxml(v);
|
|
}
|
|
return Array.from(cells, (x) => x ?? '');
|
|
});
|
|
}
|
|
|
|
async function pdfLines(path: string): Promise<string[]> {
|
|
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
|
|
const doc = await pdfjs.getDocument({ data: new Uint8Array(readFileSync(path)) }).promise;
|
|
const lines: string[] = [];
|
|
for (let p = 1; p <= doc.numPages; p++) {
|
|
const content = await (await doc.getPage(p)).getTextContent();
|
|
const byY = new Map<number, { x: number; s: string }[]>();
|
|
for (const item of content.items as { str: string; transform: number[] }[]) {
|
|
const y = Math.round(item.transform[5]);
|
|
byY.set(y, [...(byY.get(y) ?? []), { x: item.transform[4], s: item.str }]);
|
|
}
|
|
for (const y of [...byY.keys()].sort((a, b) => b - a))
|
|
lines.push(
|
|
byY
|
|
.get(y)!
|
|
.sort((a, b) => a.x - b.x)
|
|
.map((i) => i.s)
|
|
.join(' ')
|
|
);
|
|
}
|
|
return lines;
|
|
}
|
|
|
|
function loadStatement(path: string): StatementRow[] {
|
|
const lower = path.toLowerCase();
|
|
if (lower.endsWith('.xlsx')) return rowsFromTable(xlsxTable(path));
|
|
if (lower.endsWith('.csv') || lower.endsWith('.txt'))
|
|
return parseRevolutCsv(readFileSync(path, 'utf8'));
|
|
throw new Error('Use .csv, .xlsx or .pdf');
|
|
}
|
|
|
|
const all = file.toLowerCase().endsWith('.pdf')
|
|
? parseRevolutPdfLines(await pdfLines(file), Math.round(Number(value('--opening') ?? 0) * 100))
|
|
: loadStatement(file);
|
|
const { kept, skipped } = selectRows(
|
|
all,
|
|
value('--product') ?? 'Current',
|
|
value('--currency') ?? 'EUR'
|
|
);
|
|
// Chronological by booking date; file order breaks ties (it follows the running balance).
|
|
const rows = kept.slice().sort((a, b) => a.date.localeCompare(b.date) || a.line - b.line);
|
|
const balances = checkBalances(kept);
|
|
|
|
const sql = postgres(process.env.DB_URL, { max: 2, connect_timeout: 10, onnotice: () => {} });
|
|
try {
|
|
const [account] = await sql<
|
|
{
|
|
id: number;
|
|
user_id: string;
|
|
name: string;
|
|
opening_balance_cents: number;
|
|
bank_balance_cents: number | null;
|
|
bank_balance_at: Date | null;
|
|
}[]
|
|
>`select id, user_id, name, opening_balance_cents, bank_balance_cents, bank_balance_at
|
|
from money.accounts where lower(name) = lower(${accountName})`;
|
|
if (!account) throw new Error(`No tracker account named ${accountName}`);
|
|
const existing = await sql<
|
|
{
|
|
id: number;
|
|
date: string;
|
|
amount_cents: number;
|
|
description: string;
|
|
source: string;
|
|
external_id: string | null;
|
|
text: string;
|
|
}[]
|
|
>`select id, date::text, amount_cents, description, source, external_id,
|
|
concat_ws(' ', description, counterparty, raw->'creditor'->>'name', raw->'debtor'->>'name',
|
|
raw->>'remittance_information', raw->'revolut'->>'description') as text
|
|
from money.transactions where account_id = ${account.id} order by date, id`;
|
|
const startRows = existing.filter(
|
|
(t) => isOpeningBalance(t.description) && t.source === 'manual'
|
|
);
|
|
const adjustment = existing.find((t) => t.external_id === BALANCE_ADJUSTMENT_ID);
|
|
const tracker: TrackerRow[] = existing
|
|
.filter((t) => !startRows.includes(t) && t !== adjustment)
|
|
.map((t) => ({
|
|
id: t.id,
|
|
date: t.date,
|
|
amountCents: t.amount_cents,
|
|
description: t.description,
|
|
text: t.text,
|
|
source: t.source,
|
|
externalId: t.external_id
|
|
}));
|
|
const plan = planImport(rows, tracker);
|
|
const conflictsUntil = value('--insert-conflicts-until');
|
|
const toInsert = flag('--insert-conflicts')
|
|
? [...plan.fresh, ...plan.conflicts]
|
|
: conflictsUntil
|
|
? [...plan.fresh, ...plan.conflicts.filter((r) => r.date <= conflictsUntil)]
|
|
: plan.fresh;
|
|
const transferRe = value('--transfer-pattern')
|
|
? new RegExp(value('--transfer-pattern')!, 'i')
|
|
: null;
|
|
const isTransfer = (r: StatementRow) =>
|
|
!!transferRe && /transfer/i.test(r.type) && transferRe.test(r.description);
|
|
|
|
const today = new Date().toISOString().slice(0, 10);
|
|
const sum = (xs: { amountCents: number; date: string }[]) =>
|
|
xs.filter((x) => x.date <= today).reduce((s, x) => s + x.amountCents, 0);
|
|
const trackedNow =
|
|
account.opening_balance_cents +
|
|
sum(tracker) +
|
|
sum(startRows.map((t) => ({ amountCents: t.amount_cents, date: t.date }))) +
|
|
(adjustment?.amount_cents ?? 0);
|
|
const afterNoStart = account.opening_balance_cents + sum(tracker) + sum(toInsert);
|
|
const bank = account.bank_balance_cents;
|
|
const residual = bank == null ? null : bank - afterNoStart;
|
|
const trackerOnlyCents = sum(plan.trackerOnly);
|
|
const dropOk = balances.openingCents === 0 && residual !== null && residual === -trackerOnlyCents;
|
|
const boundary = plan.coverageFrom
|
|
? (rows.filter((r) => r.date < plan.coverageFrom! && r.balanceCents != null).at(-1)
|
|
?.balanceCents ?? 0)
|
|
: null;
|
|
|
|
console.log(
|
|
`Statement ${file}: ${all.length} rows; importing ${account.name} = ${value('--product') ?? 'Current'} ${value('--currency') ?? 'EUR'}: ${kept.length} completed rows ${rows[0]?.date ?? ''}..${rows.at(-1)?.date ?? ''}`
|
|
);
|
|
const skippedBy = new Map<string, { n: number; cents: number }>();
|
|
for (const r of skipped) {
|
|
const key = `${r.product}/${r.currency}/${r.state}`;
|
|
const e = skippedBy.get(key) ?? { n: 0, cents: 0 };
|
|
skippedBy.set(key, { n: e.n + 1, cents: e.cents + r.amountCents });
|
|
}
|
|
for (const [k, v] of skippedBy) console.log(` skipped ${k}: ${v.n} rows (net ${euro(v.cents)})`);
|
|
console.log(
|
|
` statement opening ${euro(balances.openingCents)}, closing ${euro(balances.closingCents)}, running-balance breaks ${balances.breaks.length}`
|
|
);
|
|
console.log(
|
|
`Tracker ${account.name} (#${account.id}): ${tracker.length} rows from ${plan.coverageFrom ?? '-'}; start balance rows ${startRows.map((t) => `#${t.id} ${t.date} ${euro(t.amount_cents)}`).join(', ') || 'none'}; adjustment ${euro(adjustment?.amount_cents)}`
|
|
);
|
|
console.log(
|
|
` statement balance just before ${plan.coverageFrom}: ${euro(boundary)} (tracker start + adjustment: ${euro(sum(startRows.map((t) => ({ amountCents: t.amount_cents, date: t.date }))) + (adjustment?.amount_cents ?? 0))})`
|
|
);
|
|
console.log(
|
|
`Plan: new ${plan.fresh.length} (${euro(sum(plan.fresh))}), duplicates ${plan.duplicates.length} (${plan.duplicates.filter((d) => !d.descMatch).length} with differing description), conflicts ${plan.conflicts.length} (${euro(sum(plan.conflicts))}), tracker-only ${plan.trackerOnly.length} (${euro(sum(plan.trackerOnly))})`
|
|
);
|
|
console.log(
|
|
`Balances: bank ${euro(bank)} (synced ${account.bank_balance_at?.toISOString() ?? 'never'}), tracker now ${euro(trackedNow)}, after import without start balance/adjustment ${euro(afterNoStart)}, gap ${euro(residual)}${residual !== null && residual === -trackerOnlyCents ? ' = tracker-only rows, OK to drop the start balance' : ` (tracker-only rows ${euro(trackerOnlyCents)}; start balance must stay)`}${transferRe ? `; transfers matching pattern: ${rows.filter(isTransfer).length}` : ''}`
|
|
);
|
|
if (flag('--list') || !apply) {
|
|
const show = (
|
|
label: string,
|
|
xs: {
|
|
date: string;
|
|
amountCents: number;
|
|
description: string;
|
|
id?: number;
|
|
source?: string;
|
|
}[],
|
|
limit: number
|
|
) => {
|
|
if (!xs.length) return;
|
|
console.log(` ${label}:`);
|
|
for (const x of xs.slice(0, flag('--list') ? Infinity : limit))
|
|
console.log(
|
|
` ${x.date} ${euro(x.amountCents).padStart(10)} ${x.description}${x.id ? ` (#${x.id} ${x.source})` : ''}`
|
|
);
|
|
if (!flag('--list') && xs.length > limit)
|
|
console.log(` … ${xs.length - limit} more (--list)`);
|
|
};
|
|
show('conflicts (statement only, in covered period)', plan.conflicts, 30);
|
|
show('tracker-only rows (covered period, no statement row)', plan.trackerOnly, 30);
|
|
show(
|
|
'duplicates with differing description',
|
|
plan.duplicates
|
|
.filter((d) => !d.descMatch)
|
|
.map((d) => ({
|
|
...d.row,
|
|
description: `${d.row.description} == ${d.match.description}`
|
|
})),
|
|
10
|
|
);
|
|
show('new rows', plan.fresh, 5);
|
|
}
|
|
if (!apply) {
|
|
console.log('Dry run: nothing written. Re-run with --apply (after a backup) to import.');
|
|
} else {
|
|
const dir = `${homedir()}/backups`;
|
|
const fresh = readdirSync(dir).some(
|
|
(f) =>
|
|
f.startsWith('money-before-') &&
|
|
f.endsWith('.dump') &&
|
|
Date.now() - statSync(`${dir}/${f}`).mtimeMs < 2 * 3600_000
|
|
);
|
|
if (!fresh)
|
|
throw new Error(`No money-before-*.dump in ${dir} from the last 2 hours; back up first`);
|
|
const drop = flag('--drop-start-balance');
|
|
if (drop && !dropOk)
|
|
throw new Error(
|
|
`Refusing --drop-start-balance: statement opening ${euro(balances.openingCents)}, gap ${euro(residual)} vs tracker-only rows ${euro(trackerOnlyCents)}`
|
|
);
|
|
const hash = (s: string) => createHash('sha256').update(s).digest('hex');
|
|
await sql.begin(async (q) => {
|
|
let inserted = 0;
|
|
for (const r of toInsert) {
|
|
const res = await q`
|
|
insert into money.transactions
|
|
(user_id, account_id, date, value_date, description, amount_cents, external_id, source, booking_status, category, raw)
|
|
values (${account.user_id}, ${account.id}, ${r.date}, ${r.startedDate}, ${r.description.slice(0, 500)},
|
|
${r.amountCents}, ${statementExternalId(r, hash)}, 'csv', 'BOOK', ${isTransfer(r) ? 'transfer' : null},
|
|
${q.json({ revolut: { type: r.type, product: r.product, started: r.startedDate, completed: r.completedAt, description: r.description, fee_cents: r.feeCents, currency: r.currency, balance_cents: r.balanceCents, file_line: r.line } })})
|
|
on conflict do nothing`;
|
|
inserted += res.count;
|
|
}
|
|
console.log(`Inserted ${inserted} rows into ${account.name}.`);
|
|
if (transferRe && flag('--tag-matched-transfers')) {
|
|
const ids = plan.duplicates.filter((d) => isTransfer(d.row)).map((d) => d.match.id);
|
|
const tagged = ids.length
|
|
? await q`update money.transactions set category = 'transfer'
|
|
where account_id = ${account.id} and id in ${q(ids)} and category is distinct from 'transfer'`
|
|
: { count: 0 };
|
|
console.log(`Tagged ${tagged.count} existing transfer rows.`);
|
|
}
|
|
if (drop) {
|
|
for (const t of startRows)
|
|
await q`delete from money.transactions where id = ${t.id} and account_id = ${account.id}`;
|
|
const needed = residual!;
|
|
if (adjustment && needed === 0)
|
|
await q`delete from money.transactions where id = ${adjustment.id}`;
|
|
else if (adjustment)
|
|
await q`update money.transactions set amount_cents = ${needed} where id = ${adjustment.id}`;
|
|
else if (needed !== 0)
|
|
await q`insert into money.transactions
|
|
(user_id, account_id, date, description, amount_cents, external_id, source, booking_status)
|
|
values (${account.user_id}, ${account.id}, ${plan.coverageFrom ?? rows[0].date}, 'Bank balance adjustment',
|
|
${needed}, ${BALANCE_ADJUSTMENT_ID}, 'enablebanking', 'ADJ')`;
|
|
const [check] = await q<{ total: number }[]>`
|
|
select (a.opening_balance_cents + coalesce(sum(t.amount_cents) filter (where t.date <= ${today}), 0))::int as total
|
|
from money.accounts a left join money.transactions t on t.account_id = a.id
|
|
where a.id = ${account.id} group by a.id`;
|
|
if (bank != null && check.total !== bank)
|
|
throw new Error(`Total would be ${euro(check.total)}, bank ${euro(bank)}; rolled back`);
|
|
console.log(
|
|
`Removed start balance ${startRows.map((t) => '#' + t.id).join(', ') || '-'}; adjustment ${euro(adjustment?.amount_cents)} -> ${euro(needed)}; total ${euro(check.total)} = bank.`
|
|
);
|
|
}
|
|
});
|
|
}
|
|
} finally {
|
|
await sql.end();
|
|
}
|