money: Enable Banking client, bank-link consent CLI, bank-sync with reconcile, systemd timer (not installed)
This commit is contained in:
411
src/lib/server/bank-sync.ts
Normal file
411
src/lib/server/bank-sync.ts
Normal file
@@ -0,0 +1,411 @@
|
||||
// Bank -> money.transactions sync (Enable Banking). Pure mapping/matching helpers are
|
||||
// exported for unit tests; the DB functions take a postgres.js `Sql` so they run the
|
||||
// same from the bun scripts (scripts/bank-link.ts, scripts/bank-sync.ts).
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { Sql } from 'postgres';
|
||||
import type {
|
||||
BankAccount,
|
||||
BankTransaction,
|
||||
EnableBankingClient,
|
||||
SessionResponse
|
||||
} from './enable-banking';
|
||||
import { EnableBankingError } from './enable-banking';
|
||||
|
||||
export type MappedTransaction = {
|
||||
externalId: string;
|
||||
date: string;
|
||||
valueDate: string | null;
|
||||
description: string;
|
||||
counterparty: string | null;
|
||||
amountCents: number;
|
||||
bookingStatus: string;
|
||||
raw: BankTransaction;
|
||||
};
|
||||
|
||||
export function amountToCents(amount: string): number {
|
||||
const raw = amount.trim();
|
||||
if (!/^-?\d+(?:\.\d+)?$/.test(raw)) throw new Error(`Invalid amount ${amount}`);
|
||||
const negative = raw.startsWith('-');
|
||||
const [whole, fraction = ''] = raw.replace('-', '').split('.');
|
||||
const cents = Number(whole) * 100 + Number((fraction + '00').slice(0, 2));
|
||||
const rounded = Number(fraction.slice(2, 3) || '0') >= 5 ? cents + 1 : cents;
|
||||
return negative ? -rounded : rounded;
|
||||
}
|
||||
|
||||
/** Stable id for a bank transaction: the bank's own reference, else a content hash. */
|
||||
export function externalIdFor(tx: BankTransaction): string {
|
||||
const ref = tx.entry_reference?.trim() || tx.transaction_id?.trim();
|
||||
if (ref) return ref;
|
||||
const fingerprint = [
|
||||
tx.booking_date ?? tx.transaction_date ?? tx.value_date ?? '',
|
||||
tx.credit_debit_indicator,
|
||||
tx.transaction_amount.amount,
|
||||
tx.transaction_amount.currency,
|
||||
tx.creditor?.name ?? '',
|
||||
tx.debtor?.name ?? '',
|
||||
(tx.remittance_information ?? []).join('|')
|
||||
].join('\u001f');
|
||||
return 'h:' + createHash('sha256').update(fingerprint).digest('hex').slice(0, 32);
|
||||
}
|
||||
|
||||
export function isBooked(tx: BankTransaction) {
|
||||
return tx.status ? tx.status === 'BOOK' : Boolean(tx.booking_date);
|
||||
}
|
||||
|
||||
/** Maps one booked bank transaction to a tracker row, or null for pending ones. */
|
||||
export function mapBankTransaction(tx: BankTransaction): MappedTransaction | null {
|
||||
if (!isBooked(tx)) return null;
|
||||
const date = tx.booking_date ?? tx.transaction_date ?? tx.value_date;
|
||||
if (!date) return null;
|
||||
const magnitude = Math.abs(amountToCents(tx.transaction_amount.amount));
|
||||
const amountCents = tx.credit_debit_indicator === 'DBIT' ? -magnitude : magnitude;
|
||||
const counterparty =
|
||||
(tx.credit_debit_indicator === 'DBIT' ? tx.creditor?.name : tx.debtor?.name)?.trim() || null;
|
||||
const remittance = (tx.remittance_information ?? []).map((line) => line.trim()).filter(Boolean);
|
||||
const description = (
|
||||
counterparty ||
|
||||
remittance.join(' ') ||
|
||||
tx.bank_transaction_code?.description ||
|
||||
tx.note ||
|
||||
(amountCents < 0 ? 'Card payment' : 'Incoming transfer')
|
||||
).slice(0, 500);
|
||||
return {
|
||||
externalId: externalIdFor(tx),
|
||||
date: date.slice(0, 10),
|
||||
valueDate: tx.value_date?.slice(0, 10) ?? null,
|
||||
description,
|
||||
counterparty,
|
||||
amountCents,
|
||||
bookingStatus: tx.status ?? 'BOOK',
|
||||
raw: tx
|
||||
};
|
||||
}
|
||||
|
||||
function dayDiff(a: string, b: string) {
|
||||
return Math.abs(Date.parse(a + 'T00:00:00Z') - Date.parse(b + 'T00:00:00Z')) / 86_400_000;
|
||||
}
|
||||
|
||||
export type ExistingRow = { id: number; date: string; amountCents: number; description: string };
|
||||
|
||||
/**
|
||||
* Finds the hand-entered row a bank transaction corresponds to: same amount, date within
|
||||
* ±1 day, not matched yet. Prefers the closest date, then a description overlap.
|
||||
*/
|
||||
export function findMatch(
|
||||
tx: Pick<MappedTransaction, 'date' | 'amountCents' | 'description'>,
|
||||
candidates: ExistingRow[],
|
||||
used: Set<number>
|
||||
): ExistingRow | null {
|
||||
const words = new Set(
|
||||
tx.description
|
||||
.toLowerCase()
|
||||
.split(/\W+/)
|
||||
.filter((w) => w.length > 2)
|
||||
);
|
||||
let best: { row: ExistingRow; score: number } | null = null;
|
||||
for (const row of candidates) {
|
||||
if (used.has(row.id) || row.amountCents !== tx.amountCents) continue;
|
||||
const diff = dayDiff(row.date, tx.date);
|
||||
if (diff > 1) continue;
|
||||
const overlap = row.description
|
||||
.toLowerCase()
|
||||
.split(/\W+/)
|
||||
.some((w) => words.has(w));
|
||||
const score = diff * 10 - (overlap ? 1 : 0) + row.id / 1e9;
|
||||
if (!best || score < best.score) best = { row, score };
|
||||
}
|
||||
return best?.row ?? null;
|
||||
}
|
||||
|
||||
export function isoDate(date: Date) {
|
||||
return date.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
export function addDays(date: string, days: number) {
|
||||
return isoDate(new Date(Date.parse(date + 'T00:00:00Z') + days * 86_400_000));
|
||||
}
|
||||
|
||||
/** Heuristic: is this bank account the joint one shared with Juhee? */
|
||||
export function looksJoint(account: BankAccount) {
|
||||
const text = [account.name, account.details, account.product].filter(Boolean).join(' ');
|
||||
return /joint|gezamenlijk|shared|\bjuhee\b|\s&\s|\sand\s|,/i.test(text);
|
||||
}
|
||||
|
||||
type TrackerAccount = {
|
||||
id: number;
|
||||
name: string;
|
||||
scope: string;
|
||||
iban: string | null;
|
||||
external_uid: string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Decides which tracker account each bank account feeds. Order: explicit override
|
||||
* (uid/IBAN -> tracker account name), same IBAN, joint heuristic -> the joint-scoped
|
||||
* account, first remaining EUR current account -> "Revolut". Others are skipped.
|
||||
*/
|
||||
export function planAccountMapping(
|
||||
bankAccounts: BankAccount[],
|
||||
tracker: TrackerAccount[],
|
||||
overrides: Record<string, string> = {}
|
||||
) {
|
||||
const byName = new Map(tracker.map((a) => [a.name.toLowerCase(), a]));
|
||||
const used = new Set<number>();
|
||||
const plan: { bank: BankAccount; tracker: TrackerAccount | null; reason: string }[] = [];
|
||||
const take = (bank: BankAccount, account: TrackerAccount | undefined | null, reason: string) => {
|
||||
if (account && !used.has(account.id)) {
|
||||
used.add(account.id);
|
||||
plan.push({ bank, tracker: account, reason });
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
const pending: BankAccount[] = [];
|
||||
for (const bank of bankAccounts) {
|
||||
const iban = bank.account_id?.iban?.replace(/\s/g, '').toUpperCase();
|
||||
const override = overrides[bank.uid ?? ''] ?? (iban ? overrides[iban] : undefined);
|
||||
if (override !== undefined) {
|
||||
if (override === '' || override === '-')
|
||||
plan.push({ bank, tracker: null, reason: 'skipped (override)' });
|
||||
else if (!take(bank, byName.get(override.toLowerCase()), 'override'))
|
||||
plan.push({ bank, tracker: null, reason: `override target "${override}" not found/used` });
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
iban &&
|
||||
take(
|
||||
bank,
|
||||
tracker.find((a) => a.iban?.replace(/\s/g, '').toUpperCase() === iban),
|
||||
'same IBAN'
|
||||
)
|
||||
)
|
||||
continue;
|
||||
pending.push(bank);
|
||||
}
|
||||
const eur = (bank: BankAccount) => !bank.currency || bank.currency === 'EUR';
|
||||
// Joint first, so a personal-looking account listed earlier can't take "Revolut" from it.
|
||||
const rest = pending.filter(
|
||||
(bank) =>
|
||||
!(
|
||||
eur(bank) &&
|
||||
looksJoint(bank) &&
|
||||
take(
|
||||
bank,
|
||||
tracker.find((a) => a.scope === 'joint'),
|
||||
'joint account'
|
||||
)
|
||||
)
|
||||
);
|
||||
for (const bank of rest) {
|
||||
if (
|
||||
eur(bank) &&
|
||||
bank.cash_account_type !== 'SVGS' &&
|
||||
take(bank, byName.get('revolut'), 'personal EUR account')
|
||||
)
|
||||
continue;
|
||||
plan.push({ bank, tracker: null, reason: 'no match (use --map uid=Account)' });
|
||||
}
|
||||
return plan;
|
||||
}
|
||||
|
||||
export async function trackerUserId(sql: Sql, explicit?: string) {
|
||||
if (explicit) return explicit;
|
||||
const rows = await sql<{ user_id: string }[]>`select distinct user_id from money.accounts`;
|
||||
if (rows.length !== 1)
|
||||
throw new Error(`Expected one money-tracker user, found ${rows.length}; pass --user <id>`);
|
||||
return rows[0].user_id;
|
||||
}
|
||||
|
||||
/** Stores a freshly authorised session and links bank accounts to tracker accounts. */
|
||||
export async function saveSession(
|
||||
sql: Sql,
|
||||
userId: string,
|
||||
session: SessionResponse,
|
||||
plan: ReturnType<typeof planAccountMapping>
|
||||
) {
|
||||
const accounts = session.accounts.map((a) => ({
|
||||
uid: a.uid ?? null,
|
||||
iban: a.account_id?.iban ?? null,
|
||||
name: a.name ?? null,
|
||||
details: a.details ?? null,
|
||||
product: a.product ?? null,
|
||||
currency: a.currency ?? null,
|
||||
cash_account_type: a.cash_account_type ?? null,
|
||||
identification_hash: a.identification_hash ?? null
|
||||
}));
|
||||
await sql.begin(async (tx) => {
|
||||
await tx`
|
||||
insert into money.bank_sessions
|
||||
(user_id, provider, session_id, aspsp_name, aspsp_country, psu_type, status, valid_until, accounts)
|
||||
values (${userId}, 'enablebanking', ${session.session_id}, ${session.aspsp.name},
|
||||
${session.aspsp.country}, ${session.psu_type}, 'authorized', ${session.access.valid_until},
|
||||
${tx.json(accounts)})
|
||||
on conflict (session_id) do update set status = 'authorized', valid_until = excluded.valid_until,
|
||||
accounts = excluded.accounts, last_error = null`;
|
||||
for (const { bank, tracker } of plan) {
|
||||
if (!tracker || !bank.uid) continue;
|
||||
// Account uids are per session: free the uid from any older mapping first.
|
||||
await tx`update money.accounts set external_uid = null where external_uid = ${bank.uid} and id <> ${tracker.id}`;
|
||||
await tx`
|
||||
update money.accounts
|
||||
set external_uid = ${bank.uid}, provider = 'enablebanking',
|
||||
iban = coalesce(${bank.account_id?.iban ?? null}, iban)
|
||||
where id = ${tracker.id} and user_id = ${userId}`;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export type SyncOptions = {
|
||||
dryRun?: boolean;
|
||||
/** Match against the whole history of hand-entered rows (one-time), not just the window. */
|
||||
reconcile?: boolean;
|
||||
from?: string;
|
||||
accountName?: string;
|
||||
log?: (line: string) => void;
|
||||
today?: string;
|
||||
};
|
||||
|
||||
export type AccountSyncResult = {
|
||||
account: string;
|
||||
fetched: number;
|
||||
booked: number;
|
||||
inserted: number;
|
||||
matched: number;
|
||||
alreadyKnown: number;
|
||||
balance?: string;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export async function syncBankAccounts(
|
||||
sql: Sql,
|
||||
client: EnableBankingClient,
|
||||
options: SyncOptions = {}
|
||||
) {
|
||||
const log = options.log ?? console.log;
|
||||
const today = options.today ?? isoDate(new Date());
|
||||
const accounts = await sql<
|
||||
{ id: number; user_id: string; name: string; external_uid: string }[]
|
||||
>`select id, user_id, name, external_uid from money.accounts
|
||||
where provider = 'enablebanking' and external_uid is not null order by id`;
|
||||
const selected = options.accountName
|
||||
? accounts.filter((a) => a.name.toLowerCase() === options.accountName!.toLowerCase())
|
||||
: accounts;
|
||||
if (!selected.length) {
|
||||
log('No linked bank accounts. Run scripts/bank-link.ts first.');
|
||||
return [];
|
||||
}
|
||||
const results: AccountSyncResult[] = [];
|
||||
for (const account of selected) {
|
||||
const result: AccountSyncResult = {
|
||||
account: account.name,
|
||||
fetched: 0,
|
||||
booked: 0,
|
||||
inserted: 0,
|
||||
matched: 0,
|
||||
alreadyKnown: 0
|
||||
};
|
||||
results.push(result);
|
||||
try {
|
||||
const [{ last }] = await sql<{ last: string | null }[]>`
|
||||
select max(date)::text as last from money.transactions
|
||||
where account_id = ${account.id} and source = 'enablebanking'`;
|
||||
const [{ first }] = await sql<{ first: string | null }[]>`
|
||||
select min(date)::text as first from money.transactions where account_id = ${account.id}`;
|
||||
const dateFrom = options.from ?? (last ? addDays(last, -7) : (first ?? addDays(today, -89)));
|
||||
let raw: BankTransaction[];
|
||||
try {
|
||||
raw = await client.transactions(account.external_uid, { dateFrom });
|
||||
} catch (error) {
|
||||
// Many banks only serve ~90 days of history after the first hour of consent.
|
||||
if (error instanceof EnableBankingError && error.code === 'WRONG_TRANSACTIONS_PERIOD') {
|
||||
log(`${account.name}: ${dateFrom} is too far back; retrying from 89 days ago`);
|
||||
raw = await client.transactions(account.external_uid, { dateFrom: addDays(today, -89) });
|
||||
} else throw error;
|
||||
}
|
||||
result.fetched = raw.length;
|
||||
const mapped = raw.map(mapBankTransaction).filter((t): t is MappedTransaction => t !== null);
|
||||
result.booked = mapped.length;
|
||||
if (!mapped.length) continue;
|
||||
|
||||
const ids = mapped.map((t) => t.externalId);
|
||||
const known = new Set(
|
||||
(
|
||||
await sql<{ external_id: string }[]>`
|
||||
select external_id from money.transactions
|
||||
where account_id = ${account.id} and external_id = any(${ids})`
|
||||
).map((r) => r.external_id)
|
||||
);
|
||||
const minDate = mapped.reduce((m, t) => (t.date < m ? t.date : m), mapped[0].date);
|
||||
const candidates = await sql<ExistingRow[]>`
|
||||
select id, date::text as date, amount_cents as "amountCents", description
|
||||
from money.transactions
|
||||
where account_id = ${account.id} and external_id is null
|
||||
and date >= ${options.reconcile ? '0001-01-01' : addDays(minDate, -1)}`;
|
||||
const used = new Set<number>();
|
||||
const inserts: MappedTransaction[] = [];
|
||||
const updates: { id: number; tx: MappedTransaction }[] = [];
|
||||
// Oldest first so matches are claimed in chronological order.
|
||||
for (const tx of [...mapped].sort((a, b) => a.date.localeCompare(b.date))) {
|
||||
if (known.has(tx.externalId)) {
|
||||
result.alreadyKnown++;
|
||||
continue;
|
||||
}
|
||||
known.add(tx.externalId);
|
||||
const match = findMatch(tx, candidates, used);
|
||||
if (match) {
|
||||
used.add(match.id);
|
||||
updates.push({ id: match.id, tx });
|
||||
} else inserts.push(tx);
|
||||
}
|
||||
result.matched = updates.length;
|
||||
result.inserted = inserts.length;
|
||||
if (!options.dryRun) {
|
||||
await sql.begin(async (q) => {
|
||||
for (const { id, tx } of updates) {
|
||||
// Keep Stan's own description/category; attach the bank identity.
|
||||
await q`
|
||||
update money.transactions set external_id = ${tx.externalId},
|
||||
counterparty = coalesce(counterparty, ${tx.counterparty}),
|
||||
booking_status = ${tx.bookingStatus}, value_date = ${tx.valueDate},
|
||||
raw = ${q.json(tx.raw as never)}
|
||||
where id = ${id} and external_id is null`;
|
||||
}
|
||||
for (const tx of inserts) {
|
||||
await q`
|
||||
insert into money.transactions
|
||||
(user_id, account_id, date, description, amount_cents, external_id, source,
|
||||
booking_status, counterparty, raw, value_date)
|
||||
values (${account.user_id}, ${account.id}, ${tx.date}, ${tx.description},
|
||||
${tx.amountCents}, ${tx.externalId}, 'enablebanking', ${tx.bookingStatus},
|
||||
${tx.counterparty}, ${q.json(tx.raw as never)}, ${tx.valueDate})
|
||||
on conflict (account_id, external_id) do nothing`;
|
||||
}
|
||||
});
|
||||
}
|
||||
try {
|
||||
const { balances } = await client.balances(account.external_uid);
|
||||
const preferred =
|
||||
balances.find((b) => b.balance_type === 'CLBD') ??
|
||||
balances.find((b) => b.balance_type === 'ITAV') ??
|
||||
balances[0];
|
||||
if (preferred)
|
||||
result.balance = `${preferred.balance_amount.amount} ${preferred.balance_amount.currency} (${preferred.balance_type})`;
|
||||
} catch {
|
||||
/* balances are informational only */
|
||||
}
|
||||
} catch (error) {
|
||||
result.error = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
}
|
||||
if (!options.dryRun) {
|
||||
const failed = results.filter((r) => r.error).map((r) => `${r.account}: ${r.error}`);
|
||||
const expired = failed.some((f) => /EXPIRED_SESSION|REVOKED_SESSION|CLOSED_SESSION/.test(f));
|
||||
await sql`
|
||||
update money.bank_sessions set last_synced_at = now(),
|
||||
last_error = ${failed.length ? failed.join('\n').slice(0, 2000) : null},
|
||||
status = case when ${expired} then 'expired'
|
||||
when valid_until is not null and valid_until < now() then 'expired' else status end
|
||||
where status = 'authorized'`;
|
||||
}
|
||||
return results;
|
||||
}
|
||||
Reference in New Issue
Block a user