fix csv import for money tracker
All checks were successful
Test and deploy staging / staging (push) Successful in 1m44s
All checks were successful
Test and deploy staging / staging (push) Successful in 1m44s
This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it } from 'vitest';
|
import { describe, expect, it } from 'vitest';
|
||||||
import { cents, csvDate, parseCsv } from './money-tracker';
|
import { cents, csvDate, importCents, parseCsv, parseTransactionImport } from './money-tracker';
|
||||||
|
|
||||||
describe('transaction import parsing', () => {
|
describe('transaction import parsing', () => {
|
||||||
it('handles quoted commas and semicolon-separated European amounts', () => {
|
it('handles quoted commas and semicolon-separated European amounts', () => {
|
||||||
@@ -15,6 +15,51 @@ describe('transaction import parsing', () => {
|
|||||||
|
|
||||||
it('rejects invalid dates and amounts rather than altering them', () => {
|
it('rejects invalid dates and amounts rather than altering them', () => {
|
||||||
expect(csvDate('31/02/2026')).toBeNull();
|
expect(csvDate('31/02/2026')).toBeNull();
|
||||||
|
expect(csvDate('Sat 1 Aug 2025')).toBeNull();
|
||||||
expect(cents('1.234,56')).toBeUndefined();
|
expect(cents('1.234,56')).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('accepts the spreadsheet export with tabs, spacer rows, named dates and euro amounts', () => {
|
||||||
|
const rows = parseCsv(
|
||||||
|
'date\taccount\tdescription\tamount\n' +
|
||||||
|
'Fri 1 Aug 2025\tDaily\tStart balance\t€123.45\n' +
|
||||||
|
'Fri 1 Aug 2025\tBusiness\tStart balance\t-€12.34\n' +
|
||||||
|
'Fri 1 Aug 2025\tShared\tStart balance\t€1,234.56\n' +
|
||||||
|
'\t\t\t\n' +
|
||||||
|
'Sat 2 Aug 2025\tShared\tExample bill\t-€1,234.50\n'
|
||||||
|
);
|
||||||
|
expect(rows).toHaveLength(5);
|
||||||
|
expect(rows[4]).toEqual(['Sat 2 Aug 2025', 'Shared', 'Example bill', '-€1,234.50']);
|
||||||
|
expect(csvDate(rows[1][0])).toBe('2025-08-01');
|
||||||
|
expect(csvDate(rows[4][0])).toBe('2025-08-02');
|
||||||
|
expect(rows.slice(1).map((row) => importCents(row[3]))).toEqual([
|
||||||
|
12345, -1234, 123456, -123450
|
||||||
|
]);
|
||||||
|
expect(
|
||||||
|
parseCsv(
|
||||||
|
'date,account,description,amount\n"Fri 1 Aug 2025",Shared,Start balance,"€1,234.56"'
|
||||||
|
)[1]
|
||||||
|
).toEqual(['Fri 1 Aug 2025', 'Shared', 'Start balance', '€1,234.56']);
|
||||||
|
expect(
|
||||||
|
parseCsv('date,account,description,amount\nFri 1 Aug 2025,Shared,Start balance,€1,234.56')[1]
|
||||||
|
).toHaveLength(5);
|
||||||
|
expect(
|
||||||
|
parseTransactionImport(
|
||||||
|
'date\taccount\tdescription\tamount\n' +
|
||||||
|
'Sat 2 Aug 2025\tShared\tExample bill\t-€1,234.50\n' +
|
||||||
|
'\t\t\t\n' +
|
||||||
|
'Fri 1 Aug 2025\tDaily\tStart balance\t€123.45\n' +
|
||||||
|
'Fri 1 Aug 2025\tBusiness\tStart balance\t-€12.34\n'
|
||||||
|
).map((row) => [row.date, row.account, row.amountCents])
|
||||||
|
).toEqual([
|
||||||
|
['2025-08-01', 'Daily', 12345],
|
||||||
|
['2025-08-01', 'Business', -1234],
|
||||||
|
['2025-08-02', 'Shared', -123450]
|
||||||
|
]);
|
||||||
|
expect(() =>
|
||||||
|
parseTransactionImport(
|
||||||
|
'date,account,description,amount\nFri 1 Aug 2025,Shared,Start balance,€1,234.56'
|
||||||
|
)
|
||||||
|
).toThrow('Quote values containing commas');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -23,6 +23,17 @@ export function cents(value: unknown, optional = false): number | null | undefin
|
|||||||
return Number.isSafeInteger(amount) && Math.abs(amount) <= 1_000_000_000 ? amount : undefined;
|
return Number.isSafeInteger(amount) && Math.abs(amount) <= 1_000_000_000 ? amount : undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function importCents(value: unknown): number | undefined {
|
||||||
|
const raw = String(value ?? '')
|
||||||
|
.replace(/[€\s\u00a0]/g, '')
|
||||||
|
.trim();
|
||||||
|
if (/^-?\d{1,3}(?:,\d{3})+(?:\.\d{1,2})?$/.test(raw))
|
||||||
|
return cents(raw.replaceAll(',', '')) ?? undefined;
|
||||||
|
if (/^-?\d{1,3}(?:\.\d{3})+(?:,\d{1,2})?$/.test(raw))
|
||||||
|
return cents(raw.replaceAll('.', '')) ?? undefined;
|
||||||
|
return cents(raw) ?? undefined;
|
||||||
|
}
|
||||||
|
|
||||||
export function validDate(value: unknown): string | null {
|
export function validDate(value: unknown): string | null {
|
||||||
const raw = String(value ?? '').trim();
|
const raw = String(value ?? '').trim();
|
||||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(raw)) return null;
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(raw)) return null;
|
||||||
@@ -49,7 +60,8 @@ export function invalid(message: string) {
|
|||||||
|
|
||||||
export function parseCsv(input: string): string[][] {
|
export function parseCsv(input: string): string[][] {
|
||||||
const rows: string[][] = [];
|
const rows: string[][] = [];
|
||||||
const delimiter = input.split(/\r?\n/, 1)[0].includes(';') ? ';' : ',';
|
const header = input.split(/\r?\n/, 1)[0];
|
||||||
|
const delimiter = header.includes('\t') ? '\t' : header.includes(';') ? ';' : ',';
|
||||||
let row: string[] = [];
|
let row: string[] = [];
|
||||||
let field = '';
|
let field = '';
|
||||||
let quoted = false;
|
let quoted = false;
|
||||||
@@ -81,7 +93,78 @@ export function csvDate(value: string): string | null {
|
|||||||
const raw = value.trim();
|
const raw = value.trim();
|
||||||
if (validDate(raw)) return raw;
|
if (validDate(raw)) return raw;
|
||||||
const match = /^(\d{1,2})[/-](\d{1,2})[/-](\d{4})$/.exec(raw);
|
const match = /^(\d{1,2})[/-](\d{1,2})[/-](\d{4})$/.exec(raw);
|
||||||
return match
|
if (match)
|
||||||
? validDate(`${match[3]}-${match[2].padStart(2, '0')}-${match[1].padStart(2, '0')}`)
|
return validDate(`${match[3]}-${match[2].padStart(2, '0')}-${match[1].padStart(2, '0')}`);
|
||||||
: null;
|
const named =
|
||||||
|
/^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+)?(\d{1,2})\s+([A-Za-z]{3,9})\s+(\d{4})$/i.exec(raw);
|
||||||
|
if (!named) return null;
|
||||||
|
const months = [
|
||||||
|
'jan',
|
||||||
|
'feb',
|
||||||
|
'mar',
|
||||||
|
'apr',
|
||||||
|
'may',
|
||||||
|
'jun',
|
||||||
|
'jul',
|
||||||
|
'aug',
|
||||||
|
'sep',
|
||||||
|
'oct',
|
||||||
|
'nov',
|
||||||
|
'dec'
|
||||||
|
];
|
||||||
|
const month = months.indexOf(named[3].slice(0, 3).toLowerCase()) + 1;
|
||||||
|
if (!month) return null;
|
||||||
|
const iso = validDate(
|
||||||
|
`${named[4]}-${String(month).padStart(2, '0')}-${named[2].padStart(2, '0')}`
|
||||||
|
);
|
||||||
|
if (!iso) return null;
|
||||||
|
if (named[1]) {
|
||||||
|
const weekdays = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat'];
|
||||||
|
if (weekdays[new Date(iso + 'T12:00:00Z').getUTCDay()] !== named[1].toLowerCase()) return null;
|
||||||
|
}
|
||||||
|
return iso;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ImportedTransaction = {
|
||||||
|
date: string;
|
||||||
|
account: string;
|
||||||
|
description: string;
|
||||||
|
amountCents: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function parseTransactionImport(input: string): ImportedTransaction[] {
|
||||||
|
const rows = parseCsv(input);
|
||||||
|
if (rows.length < 2 || rows.length > 20_001)
|
||||||
|
throw new Error('The file must have a header and 1–20,000 rows.');
|
||||||
|
const header = rows.shift()!.map((value) =>
|
||||||
|
value
|
||||||
|
.replace(/^\uFEFF/, '')
|
||||||
|
.trim()
|
||||||
|
.toLowerCase()
|
||||||
|
);
|
||||||
|
const columns = ['date', 'account', 'description', 'amount'].map((key) => header.indexOf(key));
|
||||||
|
if (columns.some((index) => index < 0))
|
||||||
|
throw new Error('Headers must include date, account, description and amount.');
|
||||||
|
const parsed: ImportedTransaction[] = [];
|
||||||
|
for (const [index, row] of rows.entries()) {
|
||||||
|
if (row.length !== header.length)
|
||||||
|
throw new Error(
|
||||||
|
`Data row ${index + 1} has ${row.length} columns; expected ${header.length}. Quote values containing commas. No rows were imported.`
|
||||||
|
);
|
||||||
|
const date = csvDate(row[columns[0]] ?? '');
|
||||||
|
const account = (row[columns[1]] ?? '').trim();
|
||||||
|
const description = (row[columns[2]] ?? '').trim();
|
||||||
|
const amountCents = importCents(row[columns[3]]);
|
||||||
|
if (
|
||||||
|
!date ||
|
||||||
|
!account ||
|
||||||
|
account.length > 80 ||
|
||||||
|
!description ||
|
||||||
|
description.length > 500 ||
|
||||||
|
amountCents == null
|
||||||
|
)
|
||||||
|
throw new Error(`Invalid data in transaction ${index + 1}. No rows were imported.`);
|
||||||
|
parsed.push({ date, account, description, amountCents });
|
||||||
|
}
|
||||||
|
return parsed.sort((a, b) => a.date.localeCompare(b.date));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,12 +11,12 @@ import {
|
|||||||
} from '$lib/server/db/schema';
|
} from '$lib/server/db/schema';
|
||||||
import { requireAdmin } from '$lib/server/authorization';
|
import { requireAdmin } from '$lib/server/authorization';
|
||||||
import {
|
import {
|
||||||
|
type ImportedTransaction,
|
||||||
cadence,
|
cadence,
|
||||||
cents,
|
cents,
|
||||||
csvDate,
|
|
||||||
invalid,
|
invalid,
|
||||||
optionalText,
|
optionalText,
|
||||||
parseCsv,
|
parseTransactionImport,
|
||||||
positiveId,
|
positiveId,
|
||||||
textField,
|
textField,
|
||||||
utilityKind,
|
utilityKind,
|
||||||
@@ -371,42 +371,13 @@ export const actions: Actions = {
|
|||||||
const form = await event.request.formData();
|
const form = await event.request.formData();
|
||||||
const file = form.get('file');
|
const file = form.get('file');
|
||||||
if (!(file instanceof File) || !file.size || file.size > 2_000_000)
|
if (!(file instanceof File) || !file.size || file.size > 2_000_000)
|
||||||
return invalid('Choose a CSV file under 2 MB.');
|
return invalid('Choose a CSV or TSV file under 2 MB.');
|
||||||
let rows: string[][];
|
let parsed: ImportedTransaction[];
|
||||||
try {
|
try {
|
||||||
rows = parseCsv(await file.text());
|
parsed = parseTransactionImport(await file.text());
|
||||||
} catch (cause) {
|
} catch (cause) {
|
||||||
return invalid(cause instanceof Error ? cause.message : 'Invalid CSV.');
|
return invalid(cause instanceof Error ? cause.message : 'Invalid CSV.');
|
||||||
}
|
}
|
||||||
if (rows.length < 2 || rows.length > 20_001)
|
|
||||||
return invalid('The CSV must have a header and 1–20,000 rows.');
|
|
||||||
const header = rows.shift()!.map((value) =>
|
|
||||||
value
|
|
||||||
.replace(/^\uFEFF/, '')
|
|
||||||
.trim()
|
|
||||||
.toLowerCase()
|
|
||||||
);
|
|
||||||
const columns = ['date', 'account', 'description', 'amount'].map((key) => header.indexOf(key));
|
|
||||||
if (columns.some((index) => index < 0))
|
|
||||||
return invalid('CSV headers must include date, account, description and amount.');
|
|
||||||
const parsed: { date: string; account: string; description: string; amountCents: number }[] =
|
|
||||||
[];
|
|
||||||
for (const [index, row] of rows.entries()) {
|
|
||||||
const date = csvDate(row[columns[0]] ?? '');
|
|
||||||
const account = (row[columns[1]] ?? '').trim();
|
|
||||||
const description = (row[columns[2]] ?? '').trim();
|
|
||||||
const amountCents = cents(row[columns[3]]);
|
|
||||||
if (
|
|
||||||
!date ||
|
|
||||||
!account ||
|
|
||||||
account.length > 80 ||
|
|
||||||
!description ||
|
|
||||||
description.length > 500 ||
|
|
||||||
amountCents == null
|
|
||||||
)
|
|
||||||
return invalid(`Invalid data in CSV row ${index + 2}. No rows were imported.`);
|
|
||||||
parsed.push({ date, account, description, amountCents });
|
|
||||||
}
|
|
||||||
await db.transaction(async (tx) => {
|
await db.transaction(async (tx) => {
|
||||||
const existing = await tx
|
const existing = await tx
|
||||||
.select()
|
.select()
|
||||||
|
|||||||
@@ -53,6 +53,7 @@
|
|||||||
let showActionError = $state(false);
|
let showActionError = $state(false);
|
||||||
let search = $state('');
|
let search = $state('');
|
||||||
let accountFilter = $state('all');
|
let accountFilter = $state('all');
|
||||||
|
let sortOrder = $state<'newest' | 'oldest'>('newest');
|
||||||
let pageNumber = $state(1);
|
let pageNumber = $state(1);
|
||||||
let selectedUtility = $state<(typeof kinds)[number]['id']>('electricity');
|
let selectedUtility = $state<(typeof kinds)[number]['id']>('electricity');
|
||||||
let dialog: HTMLDialogElement;
|
let dialog: HTMLDialogElement;
|
||||||
@@ -61,15 +62,13 @@
|
|||||||
let series = $derived(balanceSeries(data.accounts, data.transactions));
|
let series = $derived(balanceSeries(data.accounts, data.transactions));
|
||||||
let recent = $derived([...data.transactions].reverse().slice(0, 8));
|
let recent = $derived([...data.transactions].reverse().slice(0, 8));
|
||||||
let filtered = $derived(
|
let filtered = $derived(
|
||||||
[...data.transactions]
|
(sortOrder === 'newest' ? [...data.transactions].reverse() : data.transactions).filter(
|
||||||
.reverse()
|
(row) =>
|
||||||
.filter(
|
(accountFilter === 'all' || String(row.accountId) === accountFilter) &&
|
||||||
(row) =>
|
(!search ||
|
||||||
(accountFilter === 'all' || String(row.accountId) === accountFilter) &&
|
row.description.toLowerCase().includes(search.toLowerCase()) ||
|
||||||
(!search ||
|
accountName(row.accountId).toLowerCase().includes(search.toLowerCase()))
|
||||||
row.description.toLowerCase().includes(search.toLowerCase()) ||
|
)
|
||||||
accountName(row.accountId).toLowerCase().includes(search.toLowerCase()))
|
|
||||||
)
|
|
||||||
);
|
);
|
||||||
let shown = $derived(filtered.slice((pageNumber - 1) * 25, pageNumber * 25));
|
let shown = $derived(filtered.slice((pageNumber - 1) * 25, pageNumber * 25));
|
||||||
let monthlyRecurring = $derived(
|
let monthlyRecurring = $derived(
|
||||||
@@ -288,6 +287,13 @@
|
|||||||
>{account.name}</option
|
>{account.name}</option
|
||||||
>{/each}</select
|
>{/each}</select
|
||||||
>
|
>
|
||||||
|
<label class="sr-only" for="transaction-order">Transaction order</label><select
|
||||||
|
id="transaction-order"
|
||||||
|
bind:value={sortOrder}
|
||||||
|
onchange={() => (pageNumber = 1)}
|
||||||
|
><option value="newest">Newest first</option><option value="oldest">Oldest first</option
|
||||||
|
></select
|
||||||
|
>
|
||||||
</div>
|
</div>
|
||||||
<div class="table-wrap">
|
<div class="table-wrap">
|
||||||
<table>
|
<table>
|
||||||
@@ -688,9 +694,11 @@
|
|||||||
</form>
|
</form>
|
||||||
{:else if mode === 'import'}
|
{:else if mode === 'import'}
|
||||||
<p class="dialog-copy">
|
<p class="dialog-copy">
|
||||||
Upload a CSV with <code>date,account,description,amount</code> columns. Dates can be YYYY-MM-DD
|
Upload your CSV or tab-separated export with <code>date, account, description, amount</code>
|
||||||
or DD/MM/YYYY. Amounts are signed euros. New account names are created automatically. Importing
|
columns. Dates like <code>Fri 1 Aug 2025</code> and amounts like <code>-€1,234.50</code> are supported.
|
||||||
a file twice adds duplicate transactions.
|
Empty rows are skipped, and entries appear in date order. “Start balance” rows import as transactions,
|
||||||
|
so leave those accounts’ opening balances at zero. New account names are created automatically.
|
||||||
|
Importing a file twice adds duplicate transactions.
|
||||||
</p>
|
</p>
|
||||||
<form
|
<form
|
||||||
method="POST"
|
method="POST"
|
||||||
@@ -704,7 +712,12 @@
|
|||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<label class="field"
|
<label class="field"
|
||||||
>CSV file<input type="file" name="file" accept=".csv,text/csv" required /></label
|
>CSV or TSV file<input
|
||||||
|
type="file"
|
||||||
|
name="file"
|
||||||
|
accept=".csv,.tsv,text/csv,text/tab-separated-values"
|
||||||
|
required
|
||||||
|
/></label
|
||||||
>
|
>
|
||||||
<div class="dialog-footer">
|
<div class="dialog-footer">
|
||||||
<button type="button" class="button secondary" onclick={() => dialog.close()}>Cancel</button
|
<button type="button" class="button secondary" onclick={() => dialog.close()}>Cancel</button
|
||||||
@@ -745,7 +758,7 @@
|
|||||||
required
|
required
|
||||||
maxlength="80"
|
maxlength="80"
|
||||||
value={active.name ?? ''}
|
value={active.name ?? ''}
|
||||||
placeholder="Revolut"
|
placeholder="Daily"
|
||||||
/></label
|
/></label
|
||||||
>
|
>
|
||||||
<label class="field"
|
<label class="field"
|
||||||
|
|||||||
Reference in New Issue
Block a user