diff --git a/scripts/bank-link.ts b/scripts/bank-link.ts index ed7850b..33009c2 100644 --- a/scripts/bank-link.ts +++ b/scripts/bank-link.ts @@ -1,10 +1,16 @@ -// Link Revolut (personal + joint) to the money tracker through Enable Banking. +// Link a bank to the money tracker through Enable Banking. Each bank gets its own session; +// linking one never touches another bank's session. // -// bun --env-file=$HOME/.config/money-sync.env scripts/bank-link.ts interactive consent +// bun --env-file=$HOME/.config/money-sync.env scripts/bank-link.ts Revolut (personal + joint) +// ... scripts/bank-link.ts --bank abn ABN AMRO -> "ABN Amro" +// ... scripts/bank-link.ts --bank knab Knab -> "Knab" // ... scripts/bank-link.ts --check verify key + app id only // ... scripts/bank-link.ts --code '' [--yes] finish a started consent // ... scripts/bank-link.ts --map = [--map ...] override the mapping -// ... scripts/bank-link.ts --aspsp Revolut --country NL --user +// ... scripts/bank-link.ts --aspsp '' --account --country NL --user +// +// Then sync within the hour: the first sync of an account asks for the longest history the +// bank serves, and most banks only serve ~90 days once consent is older than an hour. // // Needs ENABLEBANKING_APP_ID, ENABLEBANKING_KEY_PATH, DB_URL (and optionally // ENABLEBANKING_REDIRECT_URL). Never prints the private key. @@ -33,8 +39,34 @@ args.forEach((arg, i) => { overrides[key.replace(/\s/g, '').toUpperCase()] = rest.join('='); }); +// Enable Banking ASPSP names (GET /aspsps?country=NL) and the tracker account they feed. +const BANKS: Record = { + revolut: { + aspsp: 'Revolut', + account: 'Revolut', + hint: 'Tick BOTH your personal account and the joint account with Juhee.' + }, + abn: { + aspsp: 'ABN AMRO', + account: 'ABN Amro', + hint: 'Log in with the ABN AMRO app or identifier and approve your current account.' + }, + knab: { + aspsp: 'Knab', + account: 'Knab', + hint: 'Approve your Knab current account (savings accounts are not needed).' + } +}; +const bankKey = (value('--bank') ?? 'revolut').toLowerCase().replace(/[\s-]|amro/g, ''); +const preset = value('--aspsp') && !value('--bank') ? undefined : BANKS[bankKey]; +if (!preset && !value('--aspsp')) + throw new Error(`Unknown --bank ${value('--bank')}; use ${Object.keys(BANKS).join(', ')}`); const client = createEnableBankingClient(configFromEnv()); -const aspsp = { name: value('--aspsp') ?? 'Revolut', country: value('--country') ?? 'NL' }; +const aspsp = { + name: value('--aspsp') ?? preset!.aspsp, + country: value('--country') ?? 'NL' +}; +const defaultAccount = value('--account') ?? preset?.account ?? aspsp.name; const redirectUrl = process.env.ENABLEBANKING_REDIRECT_URL ?? 'https://stanrunge.dev/projects/money-tracker/bank-callback'; @@ -47,7 +79,15 @@ const { aspsps } = await client.aspsps(aspsp.country); const bank = aspsps.find((a) => a.name.toLowerCase() === aspsp.name.toLowerCase()); if (!bank) throw new Error(`${aspsp.name} (${aspsp.country}) is not available for this app`); console.log( - `${bank.name} ${bank.country}: max consent ${Math.floor(bank.maximum_consent_validity / 86400)} days` + `${bank.name} ${bank.country}${bank.bic ? ' ' + bank.bic : ''}: max consent ` + + `${Math.floor(bank.maximum_consent_validity / 86400)} days${bank.beta ? ' (beta)' : ''}` + + (bank.required_psu_headers?.length + ? `; requires PSU headers ${bank.required_psu_headers.join(', ')} (background syncs ` + + 'run as "PSU not present", which PSD2 limits to ~4 calls a day per account)' + : '') +); +console.log( + `Linked accounts go to tracker account "${defaultAccount}" unless --map says otherwise.` ); if (flag('--check')) process.exit(0); @@ -63,8 +103,8 @@ if (!pasted) { psuType: 'personal', state: randomUUID() }); - console.log('\n1. Open this link and approve access in the Revolut app.'); - console.log(' Tick BOTH your personal account and the joint account with Juhee:\n'); + console.log(`\n1. Open this link and approve access at ${bank.name}.`); + console.log(` ${preset?.hint ?? 'Approve the account(s) you want to track.'}\n`); console.log(' ' + auth.url + '\n'); console.log(`2. You'll land on ${redirectUrl}?code=...`); console.log( @@ -88,7 +128,10 @@ try { const tracker = await sql< { id: number; name: string; scope: string; iban: string | null; external_uid: string | null }[] >`select id, name, scope, iban, external_uid from money.accounts where user_id = ${userId} order by id`; - const plan = planAccountMapping(session.accounts, tracker, overrides); + const plan = planAccountMapping(session.accounts, tracker, overrides, { + defaultAccount, + joint: bank.name.toLowerCase() === 'revolut' + }); for (const { bank: account, tracker: target, reason } of plan) { const iban = account.account_id?.iban ?? '(no IBAN)'; console.log( @@ -103,8 +146,14 @@ try { process.exit(1); } await saveSession(sql, userId, session, plan); + const linked = plan.filter((p) => p.tracker).map((p) => p.tracker!.name); + const only = linked.length === 1 ? ` --account '${linked[0]}'` : ''; + console.log('Saved. Within the next hour (full history is only served right after consent):'); console.log( - 'Saved. Next: bun --env-file=$HOME/.config/money-sync.env scripts/bank-sync.ts --reconcile --dry-run' + ` bun --env-file=$HOME/.config/money-sync.env scripts/bank-sync.ts --reconcile --dry-run${only}` + ); + console.log( + ` bun --env-file=$HOME/.config/money-sync.env scripts/bank-sync.ts --reconcile${only}` ); } finally { rl.close(); diff --git a/scripts/bank-sync.ts b/scripts/bank-sync.ts index 16c2ede..b2625b8 100644 --- a/scripts/bank-sync.ts +++ b/scripts/bank-sync.ts @@ -7,6 +7,8 @@ // run once after linking (with --dry-run first) // --from YYYY-MM-DD override the start date (default: last synced date - 7 days, or the // account's first transaction on the first run) +// --full ask for the longest history the bank serves (automatic on an +// account's first sync; only works within an hour of consent at most banks) // --account only this tracker account // // Dedupe: unique (account_id, external_id); re-running is always safe. @@ -31,12 +33,15 @@ try { dryRun, reconcile: args.includes('--reconcile'), from, + full: args.includes('--full'), accountName: value('--account') }); for (const r of results) { console.log( `${r.account}: fetched ${r.fetched}, booked ${r.booked}, new ${r.inserted}, ` + - `matched to existing ${r.matched}, already synced ${r.alreadyKnown}` + + `matched to existing ${r.matched}, replaced sum entries ${r.absorbed}, ` + + `already synced ${r.alreadyKnown}` + + (r.historyFrom ? `, history from ${r.historyFrom}` : '') + (r.balance ? `, bank balance ${r.balance}` : '') + (r.error ? `\n ERROR ${r.error}` : '') ); diff --git a/src/lib/money-tracker/charts.test.ts b/src/lib/money-tracker/charts.test.ts index b012b88..1ec3a2a 100644 --- a/src/lib/money-tracker/charts.test.ts +++ b/src/lib/money-tracker/charts.test.ts @@ -257,3 +257,27 @@ describe('money tracker calculations', () => { ).toBe(30); }); }); + +describe('history range', () => { + it('starts lines at the range start with the balance they had then', async () => { + const { clipSeries, historyStart } = await import('./charts'); + expect(historyStart('1y', '2026-10-24')).toBe('2025-10-24'); + expect(historyStart('all', '2026-10-24')).toBeNull(); + const series = [ + { + label: 'Knab', + color: 'x', + points: [ + { date: '2021-08-24', value: 100 }, + { date: '2025-01-01', value: 50 }, + { date: '2026-01-01', value: 70 } + ] + } + ]; + expect(clipSeries(series, '2025-10-24')[0].points).toEqual([ + { date: '2025-10-24', value: 50 }, + { date: '2026-01-01', value: 70 } + ]); + expect(clipSeries(series, null)).toBe(series); + }); +}); diff --git a/src/lib/money-tracker/charts.ts b/src/lib/money-tracker/charts.ts index 88e69bd..76aaf87 100644 --- a/src/lib/money-tracker/charts.ts +++ b/src/lib/money-tracker/charts.ts @@ -306,3 +306,28 @@ export function utilityEstimate( differenceCents: advancesCents - estimatedCents }; } + +export type HistoryRange = '1y' | '3y' | 'all'; + +/** First date shown for a history range (null = everything). */ +export function historyStart(range: HistoryRange, today: string): string | null { + if (range === 'all') return null; + const years = range === '1y' ? 1 : 3; + return `${Number(today.slice(0, 4)) - years}${today.slice(4)}`; +} + +/** + * Hides history before `from`, starting each line at `from` with the balance it had then, + * so a long bank history doesn't squash the recent months. + */ +export function clipSeries(series: ChartSeries[], from: string | null): ChartSeries[] { + if (!from) return series; + return series.map((line) => { + const before = line.points.filter((point) => point.date < from); + const after = line.points.filter((point) => point.date >= from); + const carried = before.at(-1); + const points = + carried && after[0]?.date !== from ? [{ date: from, value: carried.value }, ...after] : after; + return { ...line, points }; + }); +} diff --git a/src/lib/server/bank-sync.test.ts b/src/lib/server/bank-sync.test.ts index 9fc53d7..c733bd7 100644 --- a/src/lib/server/bank-sync.test.ts +++ b/src/lib/server/bank-sync.test.ts @@ -120,6 +120,22 @@ describe('account mapping', () => { ['b', 'Revolut'] ]); }); + it('maps another bank to its own tracker account without the joint heuristic', () => { + const plan = planAccountMapping( + [ + { + uid: 'n', + account_id: { iban: 'NL01ABNA0123456789' }, + name: 'S Runge & J Kim', + currency: 'EUR' + } + ], + [...tracker, { id: 7, name: 'ABN Amro', scope: 'personal', iban: null, external_uid: null }], + {}, + { defaultAccount: 'ABN Amro', joint: false } + ); + expect(plan.map((p) => [p.bank.uid, p.tracker?.name ?? null])).toEqual([['n', 'ABN Amro']]); + }); }); describe('enable banking client', () => { diff --git a/src/lib/server/bank-sync.ts b/src/lib/server/bank-sync.ts index c2d1fdb..be476a2 100644 --- a/src/lib/server/bank-sync.ts +++ b/src/lib/server/bank-sync.ts @@ -11,6 +11,7 @@ import type { SessionResponse } from './enable-banking'; import { EnableBankingError } from './enable-banking'; +import { findSumMatch } from './reconcile'; export type MappedTransaction = { externalId: string; @@ -86,6 +87,9 @@ function dayDiff(a: string, b: string) { return Math.abs(Date.parse(a + 'T00:00:00Z') - Date.parse(b + 'T00:00:00Z')) / 86_400_000; } +export const isOpeningBalance = (description: string) => + /^(start|opening) balance/i.test(description.trim()); + export type ExistingRow = { id: number; date: string; amountCents: number; description: string }; /** @@ -143,13 +147,22 @@ type TrackerAccount = { /** * Decides which tracker account each bank account feeds. Order: explicit override * (uid/IBAN -> tracker account name), same IBAN, joint heuristic -> the joint-scoped - * account, first remaining EUR current account -> "Revolut". Others are skipped. + * account (Revolut only), first remaining EUR current account -> `defaultAccount` + * ("Revolut" unless given). Others are skipped. */ export function planAccountMapping( bankAccounts: BankAccount[], tracker: TrackerAccount[], - overrides: Record = {} + overrides: Record = {}, + options: { + /** Tracker account for the first personal EUR current account (default "Revolut"). */ + defaultAccount?: string; + /** Route joint-looking accounts to the joint-scoped tracker account (Revolut only). */ + joint?: boolean; + } = {} ) { + const defaultAccount = (options.defaultAccount ?? 'Revolut').toLowerCase(); + const joint = options.joint ?? defaultAccount === 'revolut'; const byName = new Map(tracker.map((a) => [a.name.toLowerCase(), a])); const used = new Set(); const plan: { bank: BankAccount; tracker: TrackerAccount | null; reason: string }[] = []; @@ -188,6 +201,7 @@ export function planAccountMapping( const rest = pending.filter( (bank) => !( + joint && eur(bank) && looksJoint(bank) && take( @@ -201,7 +215,7 @@ export function planAccountMapping( if ( eur(bank) && bank.cash_account_type !== 'SVGS' && - take(bank, byName.get('revolut'), 'personal EUR account') + take(bank, byName.get(defaultAccount), 'personal EUR account') ) continue; plan.push({ bank, tracker: null, reason: 'no match (use --map uid=Account)' }); @@ -274,6 +288,8 @@ export type SyncOptions = { /** Match against the whole history of hand-entered rows (one-time), not just the window. */ reconcile?: boolean; from?: string; + /** Fetch the longest history the bank serves (automatic on an account's first sync). */ + full?: boolean; accountName?: string; log?: (line: string) => void; today?: string; @@ -285,7 +301,11 @@ export type AccountSyncResult = { booked: number; inserted: number; matched: number; + /** Hand-entered rows replaced because they equal the sum of 2-3 bank payments. */ + absorbed: number; alreadyKnown: number; + /** Oldest booked date the bank returned. */ + historyFrom?: string; balance?: string; /** Bank balance minus the tracker's own rows, absorbed by the adjustment row. */ adjustmentCents?: number; @@ -318,6 +338,7 @@ export async function syncBankAccounts( booked: 0, inserted: 0, matched: 0, + absorbed: 0, alreadyKnown: 0 }; results.push(result); @@ -325,22 +346,31 @@ export async function syncBankAccounts( const [{ last }] = await sql<{ last: string | null }[]>` select max(date)::text as last from money.transactions where account_id = ${account.id} and source = 'enablebanking'`; - const [{ first }] = await sql<{ first: string | null }[]>` - select min(date)::text as first from money.transactions where account_id = ${account.id}`; - const dateFrom = options.from ?? (last ? addDays(last, -7) : (first ?? addDays(today, -89))); + // First sync of an account (or --full): ask for the longest history the bank serves. + const longest = options.full || (!last && !options.from); + const dateFrom = options.from ?? (last ? addDays(last, -7) : undefined); let raw: BankTransaction[]; try { - raw = await client.transactions(account.external_uid, { dateFrom }); + raw = await client.transactions( + account.external_uid, + longest ? { strategy: 'longest', dateFrom: options.from } : { dateFrom } + ); } catch (error) { // Many banks only serve ~90 days of history after the first hour of consent. if (error instanceof EnableBankingError && error.code === 'WRONG_TRANSACTIONS_PERIOD') { - log(`${account.name}: ${dateFrom} is too far back; retrying from 89 days ago`); + log( + `${account.name}: ${dateFrom ?? 'full history'} is not available; retrying from 89 days ago` + ); raw = await client.transactions(account.external_uid, { dateFrom: addDays(today, -89) }); } else throw error; } result.fetched = raw.length; const mapped = raw.map(mapBankTransaction).filter((t): t is MappedTransaction => t !== null); result.booked = mapped.length; + result.historyFrom = mapped.reduce( + (min, t) => (!min || t.date < min ? t.date : min), + undefined + ); if (!mapped.length) { await syncBalance(sql, client, account, result, options, today, log); continue; @@ -355,11 +385,14 @@ export async function syncBankAccounts( ).map((r) => r.external_id) ); const minDate = mapped.reduce((m, t) => (t.date < m ? t.date : m), mapped[0].date); - const candidates = await sql` - select id, date::text as date, amount_cents as "amountCents", description - from money.transactions - where account_id = ${account.id} and external_id is null - and date >= ${options.reconcile ? '0001-01-01' : addDays(minDate, -1)}`; + // Opening-balance rows are not payments; never match or replace them. + const candidates = ( + await sql` + select id, date::text as date, amount_cents as "amountCents", description + from money.transactions + where account_id = ${account.id} and external_id is null + and date >= ${options.reconcile ? '0001-01-01' : addDays(minDate, -1)}` + ).filter((row) => !isOpeningBalance(row.description)); const used = new Set(); const inserts: MappedTransaction[] = []; const updates: { id: number; tx: MappedTransaction }[] = []; @@ -376,10 +409,31 @@ export async function syncBankAccounts( updates.push({ id: match.id, tx }); } else inserts.push(tx); } + // A hand entry can stand for several bank payments ("To Joint 209" = 200 + 3 + 6): + // replace it with the real payments instead of keeping both. + const absorbed: { row: ExistingRow; parts: MappedTransaction[] }[] = []; + const takenInserts = new Set(); + for (const row of candidates) { + if (used.has(row.id)) continue; + const parts = findSumMatch(row, inserts, takenInserts); + if (!parts) continue; + parts.forEach((index) => takenInserts.add(index)); + used.add(row.id); + absorbed.push({ row, parts: parts.map((index) => inserts[index]) }); + } + for (const { row, parts } of absorbed) + log( + `${account.name}: hand entry #${row.id} ${row.date} "${row.description}" ${row.amountCents} = ` + + parts.map((p) => `${p.date} ${p.description} ${p.amountCents}`).join(' + ') + + (options.dryRun ? ' (would replace)' : ' (replaced)') + ); result.matched = updates.length; + result.absorbed = absorbed.length; result.inserted = inserts.length; if (!options.dryRun) { await sql.begin(async (q) => { + for (const { row } of absorbed) + await q`delete from money.transactions where id = ${row.id} and external_id is null`; for (const { id, tx } of updates) { // Keep Stan's own description/category; attach the bank identity. await q` @@ -407,14 +461,22 @@ export async function syncBankAccounts( } } if (!options.dryRun) { - const failed = results.filter((r) => r.error).map((r) => `${r.account}: ${r.error}`); - const expired = failed.some((f) => /EXPIRED_SESSION|REVOKED_SESSION|CLOSED_SESSION/.test(f)); - await sql` - update money.bank_sessions set last_synced_at = now(), - last_error = ${failed.length ? failed.join('\n').slice(0, 2000) : null}, - status = case when ${expired} then 'expired' - when valid_until is not null and valid_until < now() then 'expired' else status end - where status = 'authorized'`; + // Each bank has its own session; only touch the sessions whose accounts were synced. + const sessions = await sql<{ id: number; accounts: { uid: string | null }[] }[]>` + select id, accounts from money.bank_sessions where status = 'authorized'`; + for (const session of sessions) { + const uids = new Set(session.accounts.map((a) => a.uid).filter(Boolean)); + const mine = results.filter((_, index) => uids.has(selected[index].external_uid)); + if (!mine.length) continue; + const failed = mine.filter((r) => r.error).map((r) => `${r.account}: ${r.error}`); + const expired = failed.some((f) => /EXPIRED_SESSION|REVOKED_SESSION|CLOSED_SESSION/.test(f)); + await sql` + update money.bank_sessions set last_synced_at = now(), + last_error = ${failed.length ? failed.join('\n').slice(0, 2000) : null}, + status = case when ${expired} then 'expired' + when valid_until is not null and valid_until < now() then 'expired' else status end + where id = ${session.id}`; + } } return results; } diff --git a/src/lib/server/enable-banking.ts b/src/lib/server/enable-banking.ts index 8233b38..69c9220 100644 --- a/src/lib/server/enable-banking.ts +++ b/src/lib/server/enable-banking.ts @@ -172,6 +172,9 @@ export function createEnableBankingClient(config: EnableBankingConfig) { country: string; maximum_consent_validity: number; psu_types: string[]; + bic?: string; + beta?: boolean; + required_psu_headers?: string[]; }[]; }>('GET', `/aspsps?${new URLSearchParams({ country, psu_type: psuType, service: 'AIS' })}`), startAuth: (input: { @@ -211,6 +214,8 @@ export function createEnableBankingClient(config: EnableBankingConfig) { dateFrom?: string; dateTo?: string; status?: 'BOOK' | 'PDNG'; + /** 'longest' asks Enable Banking to find the longest period the bank serves. */ + strategy?: 'default' | 'longest'; maxPages?: number; } = {} ) { @@ -221,6 +226,7 @@ export function createEnableBankingClient(config: EnableBankingConfig) { if (options.dateFrom) query.set('date_from', options.dateFrom); if (options.dateTo) query.set('date_to', options.dateTo); if (options.status) query.set('transaction_status', options.status); + if (options.strategy) query.set('strategy', options.strategy); if (continuationKey) query.set('continuation_key', continuationKey); const result = await request<{ transactions: BankTransaction[]; diff --git a/src/lib/server/money-tracker.test.ts b/src/lib/server/money-tracker.test.ts index 22ab1ea..73b7a2d 100644 --- a/src/lib/server/money-tracker.test.ts +++ b/src/lib/server/money-tracker.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from 'vitest'; import { + accountNumberKey, cents, csvDate, + dropInternalTransfers, importCents, parseCsv, parseTransactionImport, @@ -117,3 +119,150 @@ describe('revolut statements and import dedupe', () => { expect(duplicates).toBe(2); }); }); + +describe('knab exports', () => { + const header = + 'Rekeningnummer;Transactiedatum;Valutacode;CreditDebet;Bedrag;Tegenrekeningnummer;Tegenrekeninghouder;Valutadatum;Betaalwijze;Omschrijving;Type betaling;Machtigingsnummer;Incassant ID;Referentie;Boekdatum;\n'; + const line = ( + account: string, + date: string, + cd: string, + amount: string, + other: string, + name: string, + method: string, + note: string, + ref: string + ) => + `"${account}";"${date}";"EUR";"${cd}";"${amount}";"${other}";"${name}";"${date}";"${method}";"${note}";"";"";"";"${ref}";"${date}";\n`; + const current = + header + + line( + 'NL98KNAB0408449136', + '01-10-2026', + 'Afschrijvingen', + '1', + '80000208', + 'Knab', + 'Pakketkosten', + 'Pakketkosten', + 'R1' + ) + + line( + 'NL98KNAB0408449136', + '02-10-2026', + 'Bijschrijvingen', + '1250,5', + '30581727', + 'Spaar', + 'Ontvangen betaling', + '', + 'SHARED' + ) + + line( + 'NL98KNAB0408449136', + '03-10-2026', + 'Afschrijvingen', + '2,35', + 'NL11INGB0000000001', + 'NS', + 'iDEAL', + 'Treinkaartje', + 'R3' + ); + const savings = + header + + line( + '30581727', + '02-10-2026', + 'Afschrijvingen', + '1250,5', + 'NL98KNAB0408449136', + 'S Runge', + 'Overboeking', + '', + 'SHARED' + ) + + line('30581727', '30-09-2026', 'Bijschrijvingen', '0,42', '', '', 'Rente', 'Rente', 'R5'); + + it('parses signs, decimal commas and descriptions', () => { + const rows = parseTransactionImport(current, 'Knab'); + expect( + rows.map((row) => [row.date, row.description, row.amountCents, row.counterparty]) + ).toEqual([ + ['2026-10-01', 'Knab: Pakketkosten', -100, 'Knab'], + ['2026-10-02', 'Spaar', 125050, 'Spaar'], + ['2026-10-03', 'NS: Treinkaartje', -235, 'NS'] + ]); + expect(parseTransactionImport(current)[0].account).toBe('NL98KNAB0408449136'); + }); + + it('scopes ids to the booked account because Referentie repeats across accounts', () => { + const a = parseTransactionImport(current, 'Knab').find((row) => row.amountCents === 125050)!; + const b = parseTransactionImport(savings, 'Knab').find((row) => row.amountCents === -125050)!; + expect(a.externalId).toBe('knab:408449136:SHARED'); + expect(b.externalId).toBe('knab:30581727:SHARED'); + }); + + it('drops transfers between own accounts but keeps interest', () => { + const rows = [ + ...parseTransactionImport(current, 'Knab'), + ...parseTransactionImport(savings, 'Knab') + ]; + const { kept, internal } = dropInternalTransfers(rows, ['NL98KNAB0408449136', '30581727']); + expect(internal.map((row) => row.amountCents)).toEqual([125050, -125050]); + expect(kept.map((row) => row.amountCents)).toEqual([-100, -235, 42]); + expect(accountNumberKey('NL98 KNAB 0408 4491 36')).toBe(accountNumberKey('408449136')); + }); + + it('rejects malformed amounts', () => { + const bad = + header + + line('NL98KNAB0408449136', '01-10-2026', 'Afschrijvingen', '1.234,00', '', '', '', '', 'X'); + expect(() => parseTransactionImport(bad, 'Knab')).toThrow('Invalid data in Knab row 1'); + }); +}); + +describe('import reconcile', () => { + it('attaches hand-entered twins and replaces hand sums of several payments', () => { + const rows = [ + { accountId: 2, date: '2026-03-01', amountCents: -20000, externalId: 'k:1' }, + { accountId: 2, date: '2026-03-01', amountCents: -300, externalId: 'k:2' }, + { accountId: 2, date: '2026-03-02', amountCents: -600, externalId: 'k:3' }, + { accountId: 2, date: '2026-03-03', amountCents: 5000, externalId: 'k:4' } + ]; + const existing = [ + { + id: 1, + accountId: 2, + date: '2026-03-01', + amountCents: -20900, + externalId: null, + description: 'To Joint' + }, + { + id: 2, + accountId: 2, + date: '2026-03-03', + amountCents: 5000, + externalId: null, + description: 'From Revolut' + }, + { + id: 3, + accountId: 2, + date: '2026-03-01', + amountCents: -20300, + externalId: null, + description: 'Start balance' + } + ]; + const { insert, duplicates, attach, replace } = planImport(rows, existing); + expect(duplicates).toBe(1); + expect(attach.map((item) => [item.id, item.row.externalId])).toEqual([[2, 'k:4']]); + expect(insert.map((row) => row.externalId)).toEqual(['k:1', 'k:2', 'k:3']); + expect( + replace.map((item) => [item.id, item.parts.map((row) => row.externalId).sort()]) + ).toEqual([[1, ['k:1', 'k:2', 'k:3']]]); + }); +}); diff --git a/src/lib/server/money-tracker.ts b/src/lib/server/money-tracker.ts index 114db48..6e0e62f 100644 --- a/src/lib/server/money-tracker.ts +++ b/src/lib/server/money-tracker.ts @@ -1,5 +1,6 @@ import { fail } from '@sveltejs/kit'; import { createHash } from 'node:crypto'; +import { findSumMatch } from './reconcile'; export const utilityKinds = ['electricity', 'gas', 'water'] as const; export const cadences = ['weekly', 'monthly', 'yearly'] as const; @@ -131,11 +132,48 @@ export type ImportedTransaction = { account: string; description: string; amountCents: number; - /** Stable per-row id ("csv:") so importing the same file twice adds nothing. */ + /** Stable per-row id ("csv:", "knab::") so re-imports add nothing. */ externalId: string; + counterparty?: string | null; + /** Bank account the row was booked on (bank statements that say so, e.g. Knab). */ + sourceAccount?: string; + /** The other side's account number, used to recognise transfers between own accounts. */ + counterpartyAccount?: string; }; -export type ImportFormat = 'tracker' | 'revolut'; +export type ImportFormat = 'tracker' | 'revolut' | 'knab'; + +/** + * Canonical account number: Dutch IBANs and the bare numbers Knab uses for its savings + * accounts compare equal (NL98KNAB0408449136 ~ 408449136). + */ +export function accountNumberKey(value: string | null | undefined) { + const raw = (value ?? '').replace(/[\s.-]/g, '').toUpperCase(); + const iban = /^[A-Z]{2}\d{2}[A-Z]{4}(\d{6,})$/.exec(raw); + return (iban ? iban[1] : raw).replace(/^0+/, ''); +} + +/** Knab "Exporteer transacties" CSV (semicolon separated, Dutch headers). */ +export function isKnabHeader(header: string[]) { + return ['rekeningnummer', 'transactiedatum', 'creditdebet', 'bedrag', 'referentie'].every((key) => + header.includes(key) + ); +} + +/** Drops transfers between the given own accounts (both legs), keeping everything else. */ +export function dropInternalTransfers( + rows: T[], + own: Iterable +) { + const keys = new Set([...own].map(accountNumberKey).filter(Boolean)); + const kept: T[] = []; + const internal: T[] = []; + for (const row of rows) { + const other = accountNumberKey(row.counterpartyAccount); + (other && keys.has(other) ? internal : kept).push(row); + } + return { kept, internal }; +} function headerOf(rows: string[][]) { return rows[0].map((value) => @@ -178,6 +216,7 @@ export function parseTransactionImport( const header = headerOf(rows); rows.shift(); const seen = new Map(); + if (isKnabHeader(header)) return parseKnab(header, rows, defaultAccount, seen); if (isRevolutHeader(header)) { const account = defaultAccount?.trim(); if (!account) throw new Error('Choose which account this Revolut statement belongs to.'); @@ -249,14 +288,74 @@ export function parseTransactionImport( return parsed.sort((a, b) => a.date.localeCompare(b.date)); } +function parseKnab( + header: string[], + rows: string[][], + defaultAccount: string | undefined, + seen: Map +) { + const col = (key: string) => header.indexOf(key); + const cell = (row: string[], key: string) => (col(key) >= 0 ? (row[col(key)] ?? '').trim() : ''); + const parsed: ImportedTransaction[] = []; + for (const [index, row] of rows.entries()) { + if (row.every((value) => !value.trim())) continue; + const sourceAccount = cell(row, 'rekeningnummer').replace(/\s/g, '').toUpperCase(); + const currency = cell(row, 'valutacode').toUpperCase(); + if (currency && currency !== 'EUR') continue; + const date = csvDate(cell(row, 'transactiedatum')); + // Knab writes plain amounts with a decimal comma and never a thousands separator. + const amount = /^\d{1,9}(?:,\d{1,2})?$/.test(cell(row, 'bedrag')) + ? cents(cell(row, 'bedrag')) + : undefined; + const direction = cell(row, 'creditdebet').toLowerCase(); + const debit = direction.startsWith('af') || direction === 'd' || direction === 'debet'; + const credit = direction.startsWith('bij') || direction === 'c' || direction === 'credit'; + if (!sourceAccount || !date || amount == null || (!debit && !credit)) + throw new Error(`Invalid data in Knab row ${index + 1}. No rows were imported.`); + if (!amount) continue; + const counterparty = cell(row, 'tegenrekeninghouder'); + const note = cell(row, 'omschrijving'); + const method = cell(row, 'betaalwijze'); + const description = ( + counterparty && note && note.toLowerCase() !== counterparty.toLowerCase() + ? `${counterparty}: ${note}` + : counterparty || note || method || 'Knab' + ).slice(0, 500); + const reference = cell(row, 'referentie'); + const account = defaultAccount?.trim() || sourceAccount; + // Referentie repeats across a customer's accounts (both legs of a transfer), so the + // id is scoped to the account the row was booked on. + const externalId = reference + ? `knab:${accountNumberKey(sourceAccount)}:${reference}` + : importId(['knab', sourceAccount, date, amount, description], seen); + parsed.push({ + date, + account, + description, + amountCents: debit ? -amount : amount, + externalId, + counterparty: counterparty || null, + sourceAccount, + counterpartyAccount: cell(row, 'tegenrekeningnummer') || undefined + }); + } + if (!parsed.length) throw new Error('No EUR transactions found in this Knab export.'); + return parsed.sort((a, b) => a.date.localeCompare(b.date)); +} + export type ExistingImportRow = { id: number; accountId: number; date: string; amountCents: number; externalId: string | null; + /** Hand-entered rows (no external id) can be replaced; opening balances never. */ + description?: string; }; +const isOpeningBalanceRow = (description: string | undefined) => + /^(start|opening) balance/i.test((description ?? '').trim()); + function daysApart(a: string, b: string) { return Math.abs(Date.parse(a + 'T00:00:00Z') - Date.parse(b + 'T00:00:00Z')) / 86_400_000; } @@ -264,7 +363,10 @@ function daysApart(a: string, b: string) { /** * Splits imported rows into new ones and duplicates: same import id already stored, or an * existing row on the same account with the same amount within ±1 day (each existing row - * can only absorb one imported row). + * can only absorb one imported row). Hand-entered matches are returned in `attach` so the + * caller can give them the import id. A hand-entered row equal to the sum of 2-3 new rows + * within a day ("To Joint 209" = 200 + 3 + 6) is returned in `replace`: the new rows go in + * and the hand row should be deleted. */ export function planImport< T extends { accountId: number; date: string; amountCents: number; externalId: string } @@ -274,11 +376,13 @@ export function planImport< ); const byKey = new Map(); for (const row of existing) { + if (!row.externalId && isOpeningBalanceRow(row.description)) continue; const key = `${row.accountId}:${row.amountCents}`; byKey.set(key, [...(byKey.get(key) ?? []), row]); } const claimed = new Set(); const insert: T[] = []; + const attach: { id: number; row: T }[] = []; let duplicates = 0; for (const row of rows) { const idKey = `${row.accountId}:${row.externalId}`; @@ -292,10 +396,24 @@ export function planImport< if (match) { claimed.add(match.id); duplicates++; + if (!match.externalId) attach.push({ id: match.id, row }); continue; } ids.add(idKey); insert.push(row); } - return { insert, duplicates }; + const replace: { id: number; parts: T[] }[] = []; + const taken = new Set(); + for (const hand of existing) { + if (hand.externalId || claimed.has(hand.id) || isOpeningBalanceRow(hand.description)) continue; + const pool = insert.map((row) => + row.accountId === hand.accountId ? row : { ...row, amountCents: 0 } + ); + const parts = findSumMatch(hand, pool, taken); + if (!parts) continue; + parts.forEach((index) => taken.add(index)); + claimed.add(hand.id); + replace.push({ id: hand.id, parts: parts.map((index) => insert[index]) }); + } + return { insert, duplicates, attach, replace }; } diff --git a/src/lib/server/reconcile.test.ts b/src/lib/server/reconcile.test.ts new file mode 100644 index 0000000..b6dba07 --- /dev/null +++ b/src/lib/server/reconcile.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest'; +import { findSumMatch } from './reconcile'; + +describe('sum matching', () => { + const pool = [ + { date: '2026-03-01', amountCents: -20000 }, + { date: '2026-03-01', amountCents: -300 }, + { date: '2026-03-02', amountCents: -600 }, + { date: '2026-03-05', amountCents: -900 }, + { date: '2026-03-01', amountCents: 900 } + ]; + it('finds the payments a hand entry stands for', () => { + expect( + findSumMatch({ date: '2026-03-01', amountCents: -20900 }, pool, new Set())?.sort() + ).toEqual([0, 1, 2]); + expect(findSumMatch({ date: '2026-03-01', amountCents: -20300 }, pool, new Set())).toEqual([ + 0, 1 + ]); + }); + it('ignores used rows, other days and opposite signs', () => { + expect( + findSumMatch({ date: '2026-03-01', amountCents: -20900 }, pool, new Set([2])) + ).toBeNull(); + expect(findSumMatch({ date: '2026-03-05', amountCents: -1200 }, pool, new Set())).toBeNull(); + expect(findSumMatch({ date: '2026-03-01', amountCents: -19100 }, pool, new Set())).toBeNull(); + }); + it('needs at least two parts', () => { + expect(findSumMatch({ date: '2026-03-01', amountCents: -300 }, pool, new Set())).toBeNull(); + }); +}); diff --git a/src/lib/server/reconcile.ts b/src/lib/server/reconcile.ts new file mode 100644 index 0000000..e0ebacd --- /dev/null +++ b/src/lib/server/reconcile.ts @@ -0,0 +1,49 @@ +// Shared reconcile helpers for bank sync and CSV import: a hand-entered row often stands +// for several bank payments (e.g. "To Joint 209" = 200 + 3 + 6 on the same day). + +export type DatedAmount = { date: string; amountCents: number }; + +export function daysBetween(a: string, b: string) { + return Math.abs(Date.parse(a + 'T00:00:00Z') - Date.parse(b + 'T00:00:00Z')) / 86_400_000; +} + +/** + * Finds 2..maxParts unused pool entries, each within ±maxDays of `target.date`, whose + * amounts add up exactly to `target.amountCents`. Prefers fewer parts, then the smallest + * total date distance. Returns pool indexes, or null. Every part must have the same sign as + * the target, which keeps coincidental matches rare. + */ +export function findSumMatch( + target: DatedAmount, + pool: DatedAmount[], + used: Set, + maxParts = 3, + maxDays = 1 +): number[] | null { + const near = pool + .map((row, index) => ({ row, index, distance: daysBetween(row.date, target.date) })) + .filter( + ({ row, index, distance }) => + !used.has(index) && + Math.sign(row.amountCents) === Math.sign(target.amountCents) && + row.amountCents !== 0 && + distance <= maxDays + ); + // Same-day batches are small; cap the search so a busy day can't explode combinatorially. + if (near.length > 40) near.length = 40; + let best: { indexes: number[]; distance: number } | null = null; + const search = (size: number, start: number, chosen: typeof near, sum: number) => { + if (chosen.length === size) { + if (sum !== target.amountCents) return; + const distance = chosen.reduce((total, item) => total + item.distance, 0); + if (!best || distance < best.distance) + best = { indexes: chosen.map((item) => item.index), distance }; + return; + } + for (let i = start; i < near.length; i++) + search(size, i + 1, [...chosen, near[i]], sum + near[i].row.amountCents); + }; + for (let size = 2; size <= maxParts && !best; size++) search(size, 0, [], 0); + const found = best as { indexes: number[] } | null; + return found ? found.indexes : null; +} diff --git a/src/routes/projects/money-tracker/+page.server.ts b/src/routes/projects/money-tracker/+page.server.ts index 9545bb9..c964ba4 100644 --- a/src/routes/projects/money-tracker/+page.server.ts +++ b/src/routes/projects/money-tracker/+page.server.ts @@ -13,7 +13,9 @@ import { import { requireAdmin } from '#lib/server/authorization.js'; import { type ImportedTransaction, + accountNumberKey, cadence, + dropInternalTransfers, cents, invalid, optionalText, @@ -24,7 +26,7 @@ import { utilityKind, validDate } from '#lib/server/money-tracker.js'; -import { and, asc, desc, eq, gte, inArray } from 'drizzle-orm'; +import { and, asc, desc, eq, gte, inArray, isNull, like, sql } from 'drizzle-orm'; import { error, redirect } from '@sveltejs/kit'; import type { Actions, PageServerLoad } from './$types'; @@ -457,19 +459,34 @@ export const actions: Actions = { importTransactions: async (event) => { const user = requireAdmin(event); const form = await event.request.formData(); - const file = form.get('file'); - if (!(file instanceof File) || !file.size || file.size > 2_000_000) - return invalid('Choose a CSV or TSV file under 2 MB.'); + const files = form + .getAll('file') + .filter((file): file is File => file instanceof File && file.size > 0); + if (!files.length || files.length > 10 || files.some((file) => file.size > 2_000_000)) + return invalid('Choose up to 10 CSV or TSV files, each under 2 MB.'); const chosenId = positiveId(form.get('accountId')); const accounts = await db.select().from(moneyAccounts).where(eq(moneyAccounts.userId, user.id)); const chosen = chosenId ? accounts.find((account) => account.id === chosenId) : undefined; if (chosenId && !chosen) return invalid('Choose an account.'); - let parsed: ImportedTransaction[]; + const parsed: ImportedTransaction[] = []; try { - parsed = parseTransactionImport(await file.text(), chosen?.name); + for (const file of files) + parsed.push(...parseTransactionImport(await file.text(), chosen?.name)); } catch (cause) { return invalid(cause instanceof Error ? cause.message : 'Invalid CSV.'); } + parsed.sort((a, b) => a.date.localeCompare(b.date)); + // Bank statements that name their account go to the tracker account with that IBAN. + const byNumber = new Map( + accounts + .filter((account) => account.iban) + .map((account) => [accountNumberKey(account.iban), account]) + ); + if (!chosen) + for (const row of parsed) { + const match = row.sourceAccount && byNumber.get(accountNumberKey(row.sourceAccount)); + if (match) row.account = match.name; + } const result = await db.transaction(async (tx) => { const ids = new Map(accounts.map((account) => [account.name.toLowerCase(), account.id])); for (const name of [...new Set(parsed.map((row) => row.account))]) { @@ -485,23 +502,96 @@ export const actions: Actions = { accountId: ids.get(row.account.toLowerCase())! })); const accountIds = [...new Set(rows.map((row) => row.accountId))]; - const existing = await tx + + // Several bank accounts can feed one tracker account (Knab current + savings): transfers + // between them are internal and dropped on both sides. Interest and the rest stay. + const statementRows = await tx .select({ id: moneyTransactions.id, accountId: moneyTransactions.accountId, - date: moneyTransactions.date, - amountCents: moneyTransactions.amountCents, - externalId: moneyTransactions.externalId + externalId: moneyTransactions.externalId, + raw: moneyTransactions.raw }) .from(moneyTransactions) .where( and( eq(moneyTransactions.userId, user.id), inArray(moneyTransactions.accountId, accountIds), - gte(moneyTransactions.date, addDays(rows[0].date, -1)) + like(moneyTransactions.externalId, 'knab:%') + ) + ); + const own = new Map>(); + const addOwn = (accountId: number, number: string | null | undefined) => { + if (!number) return; + own.set(accountId, (own.get(accountId) ?? new Set()).add(accountNumberKey(number))); + }; + for (const account of accounts) + if (accountIds.includes(account.id)) addOwn(account.id, account.iban); + for (const row of rows) addOwn(row.accountId, row.sourceAccount); + for (const row of statementRows) addOwn(row.accountId, row.externalId?.split(':')[1]); + const kept: typeof rows = []; + let internal = 0; + for (const accountId of accountIds) { + const split = dropInternalTransfers( + rows.filter((row) => row.accountId === accountId), + own.get(accountId) ?? [] + ); + kept.push(...split.kept); + internal += split.internal.length; + } + kept.sort((a, b) => a.date.localeCompare(b.date)); + // Legs imported before the other account's export was known. + const stale = statementRows + .filter((row) => { + const other = accountNumberKey( + (row.raw as { counterpartyAccount?: string } | null)?.counterpartyAccount + ); + return other && own.get(row.accountId)?.has(other); + }) + .map((row) => row.id); + if (stale.length) + await tx.delete(moneyTransactions).where(inArray(moneyTransactions.id, stale)); + internal += stale.length; + if (!kept.length) return { imported: 0, duplicates: 0, internal, replaced: 0 }; + + const existing = await tx + .select({ + id: moneyTransactions.id, + accountId: moneyTransactions.accountId, + date: moneyTransactions.date, + amountCents: moneyTransactions.amountCents, + externalId: moneyTransactions.externalId, + description: moneyTransactions.description + }) + .from(moneyTransactions) + .where( + and( + eq(moneyTransactions.userId, user.id), + inArray(moneyTransactions.accountId, accountIds), + gte(moneyTransactions.date, addDays(kept[0].date, -1)) + ) + ); + const { insert, duplicates, attach, replace } = planImport(kept, existing); + // Hand-entered twins keep their description but get the import id, so re-imports skip them. + for (const { id, row } of attach) + await tx + .update(moneyTransactions) + .set({ + externalId: row.externalId, + counterparty: sql`coalesce(${moneyTransactions.counterparty}, ${row.counterparty ?? null})` + }) + .where(and(eq(moneyTransactions.id, id), isNull(moneyTransactions.externalId))); + // Hand entries that were the sum of several imported payments are replaced by them. + if (replace.length) + await tx.delete(moneyTransactions).where( + and( + inArray( + moneyTransactions.id, + replace.map((item) => item.id) + ), + isNull(moneyTransactions.externalId) ) ); - const { insert, duplicates } = planImport(rows, existing); for (let i = 0; i < insert.length; i += 500) { await tx .insert(moneyTransactions) @@ -513,12 +603,19 @@ export const actions: Actions = { description: row.description, amountCents: row.amountCents, externalId: row.externalId, + counterparty: row.counterparty ?? null, + raw: row.sourceAccount + ? { + account: row.sourceAccount, + counterpartyAccount: row.counterpartyAccount ?? null + } + : null, source: 'csv' })) ) .onConflictDoNothing(); } - return { imported: insert.length, duplicates }; + return { imported: insert.length, duplicates, internal, replaced: replace.length }; }); return result; } diff --git a/src/routes/projects/money-tracker/+page.svelte b/src/routes/projects/money-tracker/+page.svelte index cdac87d..461fc9f 100644 --- a/src/routes/projects/money-tracker/+page.svelte +++ b/src/routes/projects/money-tracker/+page.svelte @@ -25,6 +25,9 @@ import { accountBalances, accountColor, + clipSeries, + type HistoryRange, + historyStart, monthlyCents, type TimelineView, utilityEstimate, @@ -132,8 +135,17 @@ { id: 'accounts', label: 'By account' } ]; let timelineView = $state('total'); + const historyRanges: { id: HistoryRange; label: string }[] = [ + { id: '1y', label: '1 year' }, + { id: '3y', label: '3 years' }, + { id: 'all', label: 'All' } + ]; + let historyRange = $state('1y'); let timeline = $derived( - wealthTimeline(data.accounts, actualTransactions, projection, data.today, timelineView) + clipSeries( + wealthTimeline(data.accounts, actualTransactions, projection, data.today, timelineView), + historyStart(historyRange, data.today) + ) ); let projectionEnd = $derived(projection.map((line) => line.points.at(-1)?.value ?? 0)); let wishTiers = $derived( @@ -301,13 +313,22 @@ >Edit projection - {#each timelineViews as item (item.id)}{item.label}{/each} +
+ {#each timelineViews as item (item.id)}{item.label}{/each} + {#each historyRanges as item (item.id)}{item.label}{/each} +
Upload your CSV or tab-separated export with date, account, description, amount - columns, or a Revolut statement export (only completed EUR rows are imported, fees included). - Dates like Fri 1 Aug 2025 and amounts like -€1,234.50 are supported. - Empty rows are skipped, and entries appear in date order. “Start balance” rows import as transactions, - so leave those accounts’ opening balances at zero. New account names are created automatically. - Rows already imported, or already entered with the same amount within a day, are skipped. + columns, a Revolut statement export (only completed EUR rows are imported, fees included), or + Knab exports. Knab files go to the account with that IBAN; pick an account to merge several Knab + accounts (current + savings) into one, and transfers between them are left out. Dates like + Fri 1 Aug 2025 + and amounts like -€1,234.50 are supported. Empty rows are skipped, and entries appear + in date order. “Start balance” rows import as transactions, so leave those accounts’ opening balances + at zero. New account names are created automatically. Rows already imported, or already entered + with the same amount within a day, are skipped.