Files
website/src/lib/server/bank-sync.ts
Stan Runge 763d7881fb
Some checks failed
Deploy production / deploy (push) Failing after 23s
Money tracker: Knab CSV import, multi-bank linking, sum reconcile, history range
- Knab statement exports: per-account ids (knab:<account>:<Referentie>), routed by IBAN,
  transfers between own accounts merged into one tracker account are dropped (interest kept)
- Import: multiple files, hand-entered twins get the import id, hand entries equal to the
  sum of 2-3 payments within a day are replaced by them
- bank-link --bank revolut|abn|knab with per-bank default account; ASPSP notes printed
- bank-sync: longest history on an account first sync (--full to force), sum reconcile,
  per-session status so one bank failing does not mark the others
- Timeline: 1 year / 3 years / All history range
2026-10-24 01:37:40 +00:00

538 lines
20 KiB
TypeScript

// Bank -> money.transactions sync (Enable Banking). Pure mapping/matching helpers are
// exported for unit tests; the DB functions take a postgres.js `Sql` so they run the
// same from the bun scripts (scripts/bank-link.ts, scripts/bank-sync.ts).
import { createHash } from 'node:crypto';
import type { Sql } from 'postgres';
import type {
Balance,
BankAccount,
BankTransaction,
EnableBankingClient,
SessionResponse
} from './enable-banking';
import { EnableBankingError } from './enable-banking';
import { findSumMatch } from './reconcile';
export type MappedTransaction = {
externalId: string;
date: string;
valueDate: string | null;
description: string;
counterparty: string | null;
amountCents: number;
bookingStatus: string;
raw: BankTransaction;
};
export function amountToCents(amount: string): number {
const raw = amount.trim();
if (!/^-?\d+(?:\.\d+)?$/.test(raw)) throw new Error(`Invalid amount ${amount}`);
const negative = raw.startsWith('-');
const [whole, fraction = ''] = raw.replace('-', '').split('.');
const cents = Number(whole) * 100 + Number((fraction + '00').slice(0, 2));
const rounded = Number(fraction.slice(2, 3) || '0') >= 5 ? cents + 1 : cents;
return negative ? -rounded : rounded;
}
/** Stable id for a bank transaction: the bank's own reference, else a content hash. */
export function externalIdFor(tx: BankTransaction): string {
const ref = tx.entry_reference?.trim() || tx.transaction_id?.trim();
if (ref) return ref;
const fingerprint = [
tx.booking_date ?? tx.transaction_date ?? tx.value_date ?? '',
tx.credit_debit_indicator,
tx.transaction_amount.amount,
tx.transaction_amount.currency,
tx.creditor?.name ?? '',
tx.debtor?.name ?? '',
(tx.remittance_information ?? []).join('|')
].join('\u001f');
return 'h:' + createHash('sha256').update(fingerprint).digest('hex').slice(0, 32);
}
export function isBooked(tx: BankTransaction) {
return tx.status ? tx.status === 'BOOK' : Boolean(tx.booking_date);
}
/** Maps one booked bank transaction to a tracker row, or null for pending ones. */
export function mapBankTransaction(tx: BankTransaction): MappedTransaction | null {
if (!isBooked(tx)) return null;
const date = tx.booking_date ?? tx.transaction_date ?? tx.value_date;
if (!date) return null;
const magnitude = Math.abs(amountToCents(tx.transaction_amount.amount));
const amountCents = tx.credit_debit_indicator === 'DBIT' ? -magnitude : magnitude;
const counterparty =
(tx.credit_debit_indicator === 'DBIT' ? tx.creditor?.name : tx.debtor?.name)?.trim() || null;
const remittance = (tx.remittance_information ?? []).map((line) => line.trim()).filter(Boolean);
const description = (
counterparty ||
remittance.join(' ') ||
tx.bank_transaction_code?.description ||
tx.note ||
(amountCents < 0 ? 'Card payment' : 'Incoming transfer')
).slice(0, 500);
return {
externalId: externalIdFor(tx),
date: date.slice(0, 10),
valueDate: tx.value_date?.slice(0, 10) ?? null,
description,
counterparty,
amountCents,
bookingStatus: tx.status ?? 'BOOK',
raw: tx
};
}
function dayDiff(a: string, b: string) {
return Math.abs(Date.parse(a + 'T00:00:00Z') - Date.parse(b + 'T00:00:00Z')) / 86_400_000;
}
export const isOpeningBalance = (description: string) =>
/^(start|opening) balance/i.test(description.trim());
export type ExistingRow = { id: number; date: string; amountCents: number; description: string };
/**
* Finds the hand-entered row a bank transaction corresponds to: same amount, date within
* ±1 day, not matched yet. Prefers the closest date, then a description overlap.
*/
export function findMatch(
tx: Pick<MappedTransaction, 'date' | 'amountCents' | 'description'>,
candidates: ExistingRow[],
used: Set<number>
): ExistingRow | null {
const words = new Set(
tx.description
.toLowerCase()
.split(/\W+/)
.filter((w) => w.length > 2)
);
let best: { row: ExistingRow; score: number } | null = null;
for (const row of candidates) {
if (used.has(row.id) || row.amountCents !== tx.amountCents) continue;
const diff = dayDiff(row.date, tx.date);
if (diff > 1) continue;
const overlap = row.description
.toLowerCase()
.split(/\W+/)
.some((w) => words.has(w));
const score = diff * 10 - (overlap ? 1 : 0) + row.id / 1e9;
if (!best || score < best.score) best = { row, score };
}
return best?.row ?? null;
}
export function isoDate(date: Date) {
return date.toISOString().slice(0, 10);
}
export function addDays(date: string, days: number) {
return isoDate(new Date(Date.parse(date + 'T00:00:00Z') + days * 86_400_000));
}
/** Heuristic: is this bank account the joint one shared with Juhee? */
export function looksJoint(account: BankAccount) {
const text = [account.name, account.details, account.product].filter(Boolean).join(' ');
return /joint|gezamenlijk|shared|\bjuhee\b|\s&\s|\sand\s|,/i.test(text);
}
type TrackerAccount = {
id: number;
name: string;
scope: string;
iban: string | null;
external_uid: string | null;
};
/**
* Decides which tracker account each bank account feeds. Order: explicit override
* (uid/IBAN -> tracker account name), same IBAN, joint heuristic -> the joint-scoped
* account (Revolut only), first remaining EUR current account -> `defaultAccount`
* ("Revolut" unless given). Others are skipped.
*/
export function planAccountMapping(
bankAccounts: BankAccount[],
tracker: TrackerAccount[],
overrides: Record<string, string> = {},
options: {
/** Tracker account for the first personal EUR current account (default "Revolut"). */
defaultAccount?: string;
/** Route joint-looking accounts to the joint-scoped tracker account (Revolut only). */
joint?: boolean;
} = {}
) {
const defaultAccount = (options.defaultAccount ?? 'Revolut').toLowerCase();
const joint = options.joint ?? defaultAccount === 'revolut';
const byName = new Map(tracker.map((a) => [a.name.toLowerCase(), a]));
const used = new Set<number>();
const plan: { bank: BankAccount; tracker: TrackerAccount | null; reason: string }[] = [];
const take = (bank: BankAccount, account: TrackerAccount | undefined | null, reason: string) => {
if (account && !used.has(account.id)) {
used.add(account.id);
plan.push({ bank, tracker: account, reason });
return true;
}
return false;
};
const pending: BankAccount[] = [];
for (const bank of bankAccounts) {
const iban = bank.account_id?.iban?.replace(/\s/g, '').toUpperCase();
const override = overrides[bank.uid ?? ''] ?? (iban ? overrides[iban] : undefined);
if (override !== undefined) {
if (override === '' || override === '-')
plan.push({ bank, tracker: null, reason: 'skipped (override)' });
else if (!take(bank, byName.get(override.toLowerCase()), 'override'))
plan.push({ bank, tracker: null, reason: `override target "${override}" not found/used` });
continue;
}
if (
iban &&
take(
bank,
tracker.find((a) => a.iban?.replace(/\s/g, '').toUpperCase() === iban),
'same IBAN'
)
)
continue;
pending.push(bank);
}
const eur = (bank: BankAccount) => !bank.currency || bank.currency === 'EUR';
// Joint first, so a personal-looking account listed earlier can't take "Revolut" from it.
const rest = pending.filter(
(bank) =>
!(
joint &&
eur(bank) &&
looksJoint(bank) &&
take(
bank,
tracker.find((a) => a.scope === 'joint'),
'joint account'
)
)
);
for (const bank of rest) {
if (
eur(bank) &&
bank.cash_account_type !== 'SVGS' &&
take(bank, byName.get(defaultAccount), 'personal EUR account')
)
continue;
plan.push({ bank, tracker: null, reason: 'no match (use --map uid=Account)' });
}
return plan;
}
export async function trackerUserId(sql: Sql, explicit?: string) {
if (explicit) return explicit;
const rows = await sql<{ user_id: string }[]>`select distinct user_id from money.accounts`;
if (rows.length !== 1)
throw new Error(`Expected one money-tracker user, found ${rows.length}; pass --user <id>`);
return rows[0].user_id;
}
/** Stores a freshly authorised session and links bank accounts to tracker accounts. */
export async function saveSession(
sql: Sql,
userId: string,
session: SessionResponse,
plan: ReturnType<typeof planAccountMapping>
) {
const accounts = session.accounts.map((a) => ({
uid: a.uid ?? null,
iban: a.account_id?.iban ?? null,
name: a.name ?? null,
details: a.details ?? null,
product: a.product ?? null,
currency: a.currency ?? null,
cash_account_type: a.cash_account_type ?? null,
identification_hash: a.identification_hash ?? null
}));
await sql.begin(async (tx) => {
await tx`
insert into money.bank_sessions
(user_id, provider, session_id, aspsp_name, aspsp_country, psu_type, status, valid_until, accounts)
values (${userId}, 'enablebanking', ${session.session_id}, ${session.aspsp.name},
${session.aspsp.country}, ${session.psu_type}, 'authorized', ${session.access.valid_until},
${tx.json(accounts)})
on conflict (session_id) do update set status = 'authorized', valid_until = excluded.valid_until,
accounts = excluded.accounts, last_error = null`;
for (const { bank, tracker } of plan) {
if (!tracker || !bank.uid) continue;
// Account uids are per session: free the uid from any older mapping first.
await tx`update money.accounts set external_uid = null where external_uid = ${bank.uid} and id <> ${tracker.id}`;
await tx`
update money.accounts
set external_uid = ${bank.uid}, provider = 'enablebanking',
iban = coalesce(${bank.account_id?.iban ?? null}, iban)
where id = ${tracker.id} and user_id = ${userId}`;
}
});
}
/** The balance Stan sees in the bank app: interim available, else closing booked. */
export function pickBalance(balances: Balance[]): Balance | undefined {
const order = ['ITAV', 'CLAV', 'ITBD', 'CLBD', 'XPCD'];
for (const type of order) {
const found = balances.find((b) => b.balance_type === type);
if (found) return found;
}
return balances[0];
}
/** Synthetic row that makes the tracker's sum equal the bank balance. */
export const BALANCE_ADJUSTMENT_ID = 'balance-adjustment';
export type SyncOptions = {
dryRun?: boolean;
/** Match against the whole history of hand-entered rows (one-time), not just the window. */
reconcile?: boolean;
from?: string;
/** Fetch the longest history the bank serves (automatic on an account's first sync). */
full?: boolean;
accountName?: string;
log?: (line: string) => void;
today?: string;
};
export type AccountSyncResult = {
account: string;
fetched: number;
booked: number;
inserted: number;
matched: number;
/** Hand-entered rows replaced because they equal the sum of 2-3 bank payments. */
absorbed: number;
alreadyKnown: number;
/** Oldest booked date the bank returned. */
historyFrom?: string;
balance?: string;
/** Bank balance minus the tracker's own rows, absorbed by the adjustment row. */
adjustmentCents?: number;
error?: string;
};
export async function syncBankAccounts(
sql: Sql,
client: EnableBankingClient,
options: SyncOptions = {}
) {
const log = options.log ?? console.log;
const today = options.today ?? isoDate(new Date());
const accounts = await sql<
{ id: number; user_id: string; name: string; external_uid: string }[]
>`select id, user_id, name, external_uid from money.accounts
where provider = 'enablebanking' and external_uid is not null order by id`;
const selected = options.accountName
? accounts.filter((a) => a.name.toLowerCase() === options.accountName!.toLowerCase())
: accounts;
if (!selected.length) {
log('No linked bank accounts. Run scripts/bank-link.ts first.');
return [];
}
const results: AccountSyncResult[] = [];
for (const account of selected) {
const result: AccountSyncResult = {
account: account.name,
fetched: 0,
booked: 0,
inserted: 0,
matched: 0,
absorbed: 0,
alreadyKnown: 0
};
results.push(result);
try {
const [{ last }] = await sql<{ last: string | null }[]>`
select max(date)::text as last from money.transactions
where account_id = ${account.id} and source = 'enablebanking'`;
// First sync of an account (or --full): ask for the longest history the bank serves.
const longest = options.full || (!last && !options.from);
const dateFrom = options.from ?? (last ? addDays(last, -7) : undefined);
let raw: BankTransaction[];
try {
raw = await client.transactions(
account.external_uid,
longest ? { strategy: 'longest', dateFrom: options.from } : { dateFrom }
);
} catch (error) {
// Many banks only serve ~90 days of history after the first hour of consent.
if (error instanceof EnableBankingError && error.code === 'WRONG_TRANSACTIONS_PERIOD') {
log(
`${account.name}: ${dateFrom ?? 'full history'} is not available; retrying from 89 days ago`
);
raw = await client.transactions(account.external_uid, { dateFrom: addDays(today, -89) });
} else throw error;
}
result.fetched = raw.length;
const mapped = raw.map(mapBankTransaction).filter((t): t is MappedTransaction => t !== null);
result.booked = mapped.length;
result.historyFrom = mapped.reduce<string | undefined>(
(min, t) => (!min || t.date < min ? t.date : min),
undefined
);
if (!mapped.length) {
await syncBalance(sql, client, account, result, options, today, log);
continue;
}
const ids = mapped.map((t) => t.externalId);
const known = new Set(
(
await sql<{ external_id: string }[]>`
select external_id from money.transactions
where account_id = ${account.id} and external_id = any(${ids})`
).map((r) => r.external_id)
);
const minDate = mapped.reduce((m, t) => (t.date < m ? t.date : m), mapped[0].date);
// Opening-balance rows are not payments; never match or replace them.
const candidates = (
await sql<ExistingRow[]>`
select id, date::text as date, amount_cents as "amountCents", description
from money.transactions
where account_id = ${account.id} and external_id is null
and date >= ${options.reconcile ? '0001-01-01' : addDays(minDate, -1)}`
).filter((row) => !isOpeningBalance(row.description));
const used = new Set<number>();
const inserts: MappedTransaction[] = [];
const updates: { id: number; tx: MappedTransaction }[] = [];
// Oldest first so matches are claimed in chronological order.
for (const tx of [...mapped].sort((a, b) => a.date.localeCompare(b.date))) {
if (known.has(tx.externalId)) {
result.alreadyKnown++;
continue;
}
known.add(tx.externalId);
const match = findMatch(tx, candidates, used);
if (match) {
used.add(match.id);
updates.push({ id: match.id, tx });
} else inserts.push(tx);
}
// A hand entry can stand for several bank payments ("To Joint 209" = 200 + 3 + 6):
// replace it with the real payments instead of keeping both.
const absorbed: { row: ExistingRow; parts: MappedTransaction[] }[] = [];
const takenInserts = new Set<number>();
for (const row of candidates) {
if (used.has(row.id)) continue;
const parts = findSumMatch(row, inserts, takenInserts);
if (!parts) continue;
parts.forEach((index) => takenInserts.add(index));
used.add(row.id);
absorbed.push({ row, parts: parts.map((index) => inserts[index]) });
}
for (const { row, parts } of absorbed)
log(
`${account.name}: hand entry #${row.id} ${row.date} "${row.description}" ${row.amountCents} = ` +
parts.map((p) => `${p.date} ${p.description} ${p.amountCents}`).join(' + ') +
(options.dryRun ? ' (would replace)' : ' (replaced)')
);
result.matched = updates.length;
result.absorbed = absorbed.length;
result.inserted = inserts.length;
if (!options.dryRun) {
await sql.begin(async (q) => {
for (const { row } of absorbed)
await q`delete from money.transactions where id = ${row.id} and external_id is null`;
for (const { id, tx } of updates) {
// Keep Stan's own description/category; attach the bank identity.
await q`
update money.transactions set external_id = ${tx.externalId},
counterparty = coalesce(counterparty, ${tx.counterparty}),
booking_status = ${tx.bookingStatus}, value_date = ${tx.valueDate},
raw = ${q.json(tx.raw as never)}
where id = ${id} and external_id is null`;
}
for (const tx of inserts) {
await q`
insert into money.transactions
(user_id, account_id, date, description, amount_cents, external_id, source,
booking_status, counterparty, raw, value_date)
values (${account.user_id}, ${account.id}, ${tx.date}, ${tx.description},
${tx.amountCents}, ${tx.externalId}, 'enablebanking', ${tx.bookingStatus},
${tx.counterparty}, ${q.json(tx.raw as never)}, ${tx.valueDate})
on conflict (account_id, external_id) do nothing`;
}
});
}
await syncBalance(sql, client, account, result, options, today, log);
} catch (error) {
result.error = error instanceof Error ? error.message : String(error);
}
}
if (!options.dryRun) {
// Each bank has its own session; only touch the sessions whose accounts were synced.
const sessions = await sql<{ id: number; accounts: { uid: string | null }[] }[]>`
select id, accounts from money.bank_sessions where status = 'authorized'`;
for (const session of sessions) {
const uids = new Set(session.accounts.map((a) => a.uid).filter(Boolean));
const mine = results.filter((_, index) => uids.has(selected[index].external_uid));
if (!mine.length) continue;
const failed = mine.filter((r) => r.error).map((r) => `${r.account}: ${r.error}`);
const expired = failed.some((f) => /EXPIRED_SESSION|REVOKED_SESSION|CLOSED_SESSION/.test(f));
await sql`
update money.bank_sessions set last_synced_at = now(),
last_error = ${failed.length ? failed.join('\n').slice(0, 2000) : null},
status = case when ${expired} then 'expired'
when valid_until is not null and valid_until < now() then 'expired' else status end
where id = ${session.id}`;
}
}
return results;
}
async function syncBalance(
sql: Sql,
client: EnableBankingClient,
account: { id: number; user_id: string; name: string; external_uid: string },
result: AccountSyncResult,
options: SyncOptions,
today: string,
log: (line: string) => void
) {
let balances: Balance[];
try {
({ balances } = await client.balances(account.external_uid));
} catch (error) {
log(`${account.name}: balance unavailable (${error instanceof Error ? error.message : error})`);
return;
}
const chosen = pickBalance(balances);
if (!chosen) return;
const bankCents = amountToCents(chosen.balance_amount.amount);
result.balance = `${chosen.balance_amount.amount} ${chosen.balance_amount.currency} (${chosen.balance_type})`;
await sql.begin(async (q) => {
const [row] = await q<{ tracked: number; adjustment: number | null; first: string | null }[]>`
select (a.opening_balance_cents + coalesce(sum(t.amount_cents) filter (
where t.date <= ${today} and t.external_id is distinct from ${BALANCE_ADJUSTMENT_ID}), 0))::int as tracked,
(select amount_cents from money.transactions
where account_id = a.id and external_id = ${BALANCE_ADJUSTMENT_ID}) as adjustment,
min(t.date)::text as first
from money.accounts a left join money.transactions t on t.account_id = a.id
where a.id = ${account.id} group by a.id`;
const needed = bankCents - row.tracked;
result.adjustmentCents = needed;
if (row.adjustment !== needed)
log(
`${account.name}: tracker rows sum differs from bank by ${(needed / 100).toFixed(2)}; adjustment ${row.adjustment ?? 'none'} -> ${needed}`
);
if (options.dryRun) return;
await q`
update money.accounts set bank_balance_cents = ${bankCents},
bank_balance_type = ${chosen.balance_type}, bank_balance_at = now()
where id = ${account.id}`;
if (row.adjustment === null) {
if (needed !== 0)
await q`
insert into money.transactions
(user_id, account_id, date, description, amount_cents, external_id, source, booking_status)
values (${account.user_id}, ${account.id}, ${row.first ?? today}, 'Bank balance adjustment',
${needed}, ${BALANCE_ADJUSTMENT_ID}, 'enablebanking', 'ADJ')`;
} else if (row.adjustment !== needed) {
await q`
update money.transactions set amount_cents = ${needed}
where account_id = ${account.id} and external_id = ${BALANCE_ADJUSTMENT_ID}`;
}
});
}