fix csv import for money tracker
All checks were successful
Test and deploy staging / staging (push) Successful in 1m44s

This commit is contained in:
2026-09-23 17:23:14 +00:00
parent 0638ead87f
commit 2466db74cd
4 changed files with 165 additions and 53 deletions

View File

@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest'; import { describe, expect, it } from 'vitest';
import { cents, csvDate, parseCsv } from './money-tracker'; import { cents, csvDate, importCents, parseCsv, parseTransactionImport } from './money-tracker';
describe('transaction import parsing', () => { describe('transaction import parsing', () => {
it('handles quoted commas and semicolon-separated European amounts', () => { it('handles quoted commas and semicolon-separated European amounts', () => {
@@ -15,6 +15,51 @@ describe('transaction import parsing', () => {
it('rejects invalid dates and amounts rather than altering them', () => { it('rejects invalid dates and amounts rather than altering them', () => {
expect(csvDate('31/02/2026')).toBeNull(); expect(csvDate('31/02/2026')).toBeNull();
expect(csvDate('Sat 1 Aug 2025')).toBeNull();
expect(cents('1.234,56')).toBeUndefined(); expect(cents('1.234,56')).toBeUndefined();
}); });
it('accepts the spreadsheet export with tabs, spacer rows, named dates and euro amounts', () => {
const rows = parseCsv(
'date\taccount\tdescription\tamount\n' +
'Fri 1 Aug 2025\tDaily\tStart balance\t€123.45\n' +
'Fri 1 Aug 2025\tBusiness\tStart balance\t-€12.34\n' +
'Fri 1 Aug 2025\tShared\tStart balance\t€1,234.56\n' +
'\t\t\t\n' +
'Sat 2 Aug 2025\tShared\tExample bill\t-€1,234.50\n'
);
expect(rows).toHaveLength(5);
expect(rows[4]).toEqual(['Sat 2 Aug 2025', 'Shared', 'Example bill', '-€1,234.50']);
expect(csvDate(rows[1][0])).toBe('2025-08-01');
expect(csvDate(rows[4][0])).toBe('2025-08-02');
expect(rows.slice(1).map((row) => importCents(row[3]))).toEqual([
12345, -1234, 123456, -123450
]);
expect(
parseCsv(
'date,account,description,amount\n"Fri 1 Aug 2025",Shared,Start balance,"€1,234.56"'
)[1]
).toEqual(['Fri 1 Aug 2025', 'Shared', 'Start balance', '€1,234.56']);
expect(
parseCsv('date,account,description,amount\nFri 1 Aug 2025,Shared,Start balance,€1,234.56')[1]
).toHaveLength(5);
expect(
parseTransactionImport(
'date\taccount\tdescription\tamount\n' +
'Sat 2 Aug 2025\tShared\tExample bill\t-€1,234.50\n' +
'\t\t\t\n' +
'Fri 1 Aug 2025\tDaily\tStart balance\t€123.45\n' +
'Fri 1 Aug 2025\tBusiness\tStart balance\t-€12.34\n'
).map((row) => [row.date, row.account, row.amountCents])
).toEqual([
['2025-08-01', 'Daily', 12345],
['2025-08-01', 'Business', -1234],
['2025-08-02', 'Shared', -123450]
]);
expect(() =>
parseTransactionImport(
'date,account,description,amount\nFri 1 Aug 2025,Shared,Start balance,€1,234.56'
)
).toThrow('Quote values containing commas');
});
}); });

View File

@@ -23,6 +23,17 @@ export function cents(value: unknown, optional = false): number | null | undefin
return Number.isSafeInteger(amount) && Math.abs(amount) <= 1_000_000_000 ? amount : undefined; return Number.isSafeInteger(amount) && Math.abs(amount) <= 1_000_000_000 ? amount : undefined;
} }
export function importCents(value: unknown): number | undefined {
const raw = String(value ?? '')
.replace(/[€\s\u00a0]/g, '')
.trim();
if (/^-?\d{1,3}(?:,\d{3})+(?:\.\d{1,2})?$/.test(raw))
return cents(raw.replaceAll(',', '')) ?? undefined;
if (/^-?\d{1,3}(?:\.\d{3})+(?:,\d{1,2})?$/.test(raw))
return cents(raw.replaceAll('.', '')) ?? undefined;
return cents(raw) ?? undefined;
}
export function validDate(value: unknown): string | null { export function validDate(value: unknown): string | null {
const raw = String(value ?? '').trim(); const raw = String(value ?? '').trim();
if (!/^\d{4}-\d{2}-\d{2}$/.test(raw)) return null; if (!/^\d{4}-\d{2}-\d{2}$/.test(raw)) return null;
@@ -49,7 +60,8 @@ export function invalid(message: string) {
export function parseCsv(input: string): string[][] { export function parseCsv(input: string): string[][] {
const rows: string[][] = []; const rows: string[][] = [];
const delimiter = input.split(/\r?\n/, 1)[0].includes(';') ? ';' : ','; const header = input.split(/\r?\n/, 1)[0];
const delimiter = header.includes('\t') ? '\t' : header.includes(';') ? ';' : ',';
let row: string[] = []; let row: string[] = [];
let field = ''; let field = '';
let quoted = false; let quoted = false;
@@ -81,7 +93,78 @@ export function csvDate(value: string): string | null {
const raw = value.trim(); const raw = value.trim();
if (validDate(raw)) return raw; if (validDate(raw)) return raw;
const match = /^(\d{1,2})[/-](\d{1,2})[/-](\d{4})$/.exec(raw); const match = /^(\d{1,2})[/-](\d{1,2})[/-](\d{4})$/.exec(raw);
return match if (match)
? validDate(`${match[3]}-${match[2].padStart(2, '0')}-${match[1].padStart(2, '0')}`) return validDate(`${match[3]}-${match[2].padStart(2, '0')}-${match[1].padStart(2, '0')}`);
: null; const named =
/^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+)?(\d{1,2})\s+([A-Za-z]{3,9})\s+(\d{4})$/i.exec(raw);
if (!named) return null;
const months = [
'jan',
'feb',
'mar',
'apr',
'may',
'jun',
'jul',
'aug',
'sep',
'oct',
'nov',
'dec'
];
const month = months.indexOf(named[3].slice(0, 3).toLowerCase()) + 1;
if (!month) return null;
const iso = validDate(
`${named[4]}-${String(month).padStart(2, '0')}-${named[2].padStart(2, '0')}`
);
if (!iso) return null;
if (named[1]) {
const weekdays = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat'];
if (weekdays[new Date(iso + 'T12:00:00Z').getUTCDay()] !== named[1].toLowerCase()) return null;
}
return iso;
}
export type ImportedTransaction = {
date: string;
account: string;
description: string;
amountCents: number;
};
export function parseTransactionImport(input: string): ImportedTransaction[] {
const rows = parseCsv(input);
if (rows.length < 2 || rows.length > 20_001)
throw new Error('The file must have a header and 1–20,000 rows.');
const header = rows.shift()!.map((value) =>
value
.replace(/^\uFEFF/, '')
.trim()
.toLowerCase()
);
const columns = ['date', 'account', 'description', 'amount'].map((key) => header.indexOf(key));
if (columns.some((index) => index < 0))
throw new Error('Headers must include date, account, description and amount.');
const parsed: ImportedTransaction[] = [];
for (const [index, row] of rows.entries()) {
if (row.length !== header.length)
throw new Error(
`Data row ${index + 1} has ${row.length} columns; expected ${header.length}. Quote values containing commas. No rows were imported.`
);
const date = csvDate(row[columns[0]] ?? '');
const account = (row[columns[1]] ?? '').trim();
const description = (row[columns[2]] ?? '').trim();
const amountCents = importCents(row[columns[3]]);
if (
!date ||
!account ||
account.length > 80 ||
!description ||
description.length > 500 ||
amountCents == null
)
throw new Error(`Invalid data in transaction ${index + 1}. No rows were imported.`);
parsed.push({ date, account, description, amountCents });
}
return parsed.sort((a, b) => a.date.localeCompare(b.date));
} }

View File

@@ -11,12 +11,12 @@ import {
} from '$lib/server/db/schema'; } from '$lib/server/db/schema';
import { requireAdmin } from '$lib/server/authorization'; import { requireAdmin } from '$lib/server/authorization';
import { import {
type ImportedTransaction,
cadence, cadence,
cents, cents,
csvDate,
invalid, invalid,
optionalText, optionalText,
parseCsv, parseTransactionImport,
positiveId, positiveId,
textField, textField,
utilityKind, utilityKind,
@@ -371,42 +371,13 @@ export const actions: Actions = {
const form = await event.request.formData(); const form = await event.request.formData();
const file = form.get('file'); const file = form.get('file');
if (!(file instanceof File) || !file.size || file.size > 2_000_000) if (!(file instanceof File) || !file.size || file.size > 2_000_000)
return invalid('Choose a CSV file under 2 MB.'); return invalid('Choose a CSV or TSV file under 2 MB.');
let rows: string[][]; let parsed: ImportedTransaction[];
try { try {
rows = parseCsv(await file.text()); parsed = parseTransactionImport(await file.text());
} catch (cause) { } catch (cause) {
return invalid(cause instanceof Error ? cause.message : 'Invalid CSV.'); return invalid(cause instanceof Error ? cause.message : 'Invalid CSV.');
} }
if (rows.length < 2 || rows.length > 20_001)
return invalid('The CSV must have a header and 1–20,000 rows.');
const header = rows.shift()!.map((value) =>
value
.replace(/^\uFEFF/, '')
.trim()
.toLowerCase()
);
const columns = ['date', 'account', 'description', 'amount'].map((key) => header.indexOf(key));
if (columns.some((index) => index < 0))
return invalid('CSV headers must include date, account, description and amount.');
const parsed: { date: string; account: string; description: string; amountCents: number }[] =
[];
for (const [index, row] of rows.entries()) {
const date = csvDate(row[columns[0]] ?? '');
const account = (row[columns[1]] ?? '').trim();
const description = (row[columns[2]] ?? '').trim();
const amountCents = cents(row[columns[3]]);
if (
!date ||
!account ||
account.length > 80 ||
!description ||
description.length > 500 ||
amountCents == null
)
return invalid(`Invalid data in CSV row ${index + 2}. No rows were imported.`);
parsed.push({ date, account, description, amountCents });
}
await db.transaction(async (tx) => { await db.transaction(async (tx) => {
const existing = await tx const existing = await tx
.select() .select()

View File

@@ -53,6 +53,7 @@
let showActionError = $state(false); let showActionError = $state(false);
let search = $state(''); let search = $state('');
let accountFilter = $state('all'); let accountFilter = $state('all');
let sortOrder = $state<'newest' | 'oldest'>('newest');
let pageNumber = $state(1); let pageNumber = $state(1);
let selectedUtility = $state<(typeof kinds)[number]['id']>('electricity'); let selectedUtility = $state<(typeof kinds)[number]['id']>('electricity');
let dialog: HTMLDialogElement; let dialog: HTMLDialogElement;
@@ -61,15 +62,13 @@
let series = $derived(balanceSeries(data.accounts, data.transactions)); let series = $derived(balanceSeries(data.accounts, data.transactions));
let recent = $derived([...data.transactions].reverse().slice(0, 8)); let recent = $derived([...data.transactions].reverse().slice(0, 8));
let filtered = $derived( let filtered = $derived(
[...data.transactions] (sortOrder === 'newest' ? [...data.transactions].reverse() : data.transactions).filter(
.reverse() (row) =>
.filter( (accountFilter === 'all' || String(row.accountId) === accountFilter) &&
(row) => (!search ||
(accountFilter === 'all' || String(row.accountId) === accountFilter) && row.description.toLowerCase().includes(search.toLowerCase()) ||
(!search || accountName(row.accountId).toLowerCase().includes(search.toLowerCase()))
row.description.toLowerCase().includes(search.toLowerCase()) || )
accountName(row.accountId).toLowerCase().includes(search.toLowerCase()))
)
); );
let shown = $derived(filtered.slice((pageNumber - 1) * 25, pageNumber * 25)); let shown = $derived(filtered.slice((pageNumber - 1) * 25, pageNumber * 25));
let monthlyRecurring = $derived( let monthlyRecurring = $derived(
@@ -288,6 +287,13 @@
>{account.name}</option >{account.name}</option
>{/each}</select >{/each}</select
> >
<label class="sr-only" for="transaction-order">Transaction order</label><select
id="transaction-order"
bind:value={sortOrder}
onchange={() => (pageNumber = 1)}
><option value="newest">Newest first</option><option value="oldest">Oldest first</option
></select
>
</div> </div>
<div class="table-wrap"> <div class="table-wrap">
<table> <table>
@@ -688,9 +694,11 @@
</form> </form>
{:else if mode === 'import'} {:else if mode === 'import'}
<p class="dialog-copy"> <p class="dialog-copy">
Upload a CSV with <code>date,account,description,amount</code> columns. Dates can be YYYY-MM-DD Upload your CSV or tab-separated export with <code>date, account, description, amount</code>
or DD/MM/YYYY. Amounts are signed euros. New account names are created automatically. Importing columns. Dates like <code>Fri 1 Aug 2025</code> and amounts like <code>-€1,234.50</code> are supported.
a file twice adds duplicate transactions. Empty rows are skipped, and entries appear in date order. “Start balance” rows import as transactions,
so leave those accounts’ opening balances at zero. New account names are created automatically.
Importing a file twice adds duplicate transactions.
</p> </p>
<form <form
method="POST" method="POST"
@@ -704,7 +712,12 @@
}} }}
> >
<label class="field" <label class="field"
>CSV file<input type="file" name="file" accept=".csv,text/csv" required /></label >CSV or TSV file<input
type="file"
name="file"
accept=".csv,.tsv,text/csv,text/tab-separated-values"
required
/></label
> >
<div class="dialog-footer"> <div class="dialog-footer">
<button type="button" class="button secondary" onclick={() => dialog.close()}>Cancel</button <button type="button" class="button secondary" onclick={() => dialog.close()}>Cancel</button
@@ -745,7 +758,7 @@
required required
maxlength="80" maxlength="80"
value={active.name ?? ''} value={active.name ?? ''}
placeholder="Revolut" placeholder="Daily"
/></label /></label
> >
<label class="field" <label class="field"