Stan Runge
772e699655
Some checks failed
Deploy production / deploy (push) Has been cancelled
dataholders schema (orgs + identity) with migration and personal_app grants, idempotent CSV import (upsert on name+login, never wipes, keeps site edits and ticks), owner-only /trackers/data-holders with search, filters, pagination, inline edit/add and a moving checklist per identifier
62 lines
2.5 KiB
TypeScript
62 lines
2.5 KiB
TypeScript
/**
|
|
* Import organisations that hold data about Stan into the private `dataholders` schema.
|
|
* Idempotent: re-run it whenever the CSV grows. Never deletes rows, never touches checklist
|
|
* ticks or archive flags, and only fills empty fields on rows edited on the site.
|
|
*
|
|
* bun scripts/dataholders-import.ts data.csv # import
|
|
* bun scripts/dataholders-import.ts --dry-run data.csv # validate and show what would change
|
|
* bun scripts/dataholders-import.ts --address "Street 1, 1234 AB City" --since 2026-10-15 data.csv
|
|
*
|
|
* Columns (header names, any order): name, category, data_held, address_on_file,
|
|
* email_on_file, phone_on_file, action, how_to_change, link, auto_from_brp, priority, status,
|
|
* evidence, notes, last_checked, source, login. A `password` column is refused.
|
|
* Uses DB_URL (the app role on prod, personal_dev on dev).
|
|
*/
|
|
import { readFile } from 'node:fs/promises';
|
|
import postgres from 'postgres';
|
|
import {
|
|
importOrgs,
|
|
mapCsvRow,
|
|
parseCsv,
|
|
seedIdentity
|
|
} from '../src/lib/server/dataholders-import';
|
|
|
|
const args = process.argv.slice(2);
|
|
const opt = (name: string) => {
|
|
const i = args.indexOf(name);
|
|
return i >= 0 ? args.splice(i, 2)[1] : undefined;
|
|
};
|
|
const address = opt('--address');
|
|
const since = opt('--since');
|
|
const dryRun = args.includes('--dry-run');
|
|
const files = args.filter((a) => !a.startsWith('--'));
|
|
if (files.length !== 1) {
|
|
console.error(
|
|
'usage: bun scripts/dataholders-import.ts [--dry-run] [--address A --since YYYY-MM-DD] <orgs.csv>'
|
|
);
|
|
process.exit(2);
|
|
}
|
|
const records = parseCsv(await readFile(files[0], 'utf8'));
|
|
if (records.length && Object.keys(records[0]).some((k) => /pass(word)?|secret|otp|totp/i.test(k))) {
|
|
console.error('Refusing: the CSV has a password/secret column. Remove it first.');
|
|
process.exit(1);
|
|
}
|
|
const rows = records.map(mapCsvRow).filter((r) => r !== null);
|
|
console.log(`Valid: ${rows.length} of ${records.length} rows`);
|
|
if (!process.env.DB_URL) throw new Error('DB_URL is required');
|
|
const sql = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
|
|
try {
|
|
if (address && !dryRun)
|
|
await seedIdentity(sql, [
|
|
{ kind: 'address', value: address, since: since ?? new Date().toISOString().slice(0, 10) }
|
|
]);
|
|
const r = await importOrgs(sql, rows, { dryRun });
|
|
console.log(
|
|
`${dryRun ? '[dry run] ' : ''}${r.inserted} inserted, ${r.updated} updated, ${r.unchanged} unchanged, ` +
|
|
`${r.linkedLogin} logins attached to existing orgs` +
|
|
(r.duplicates.length ? `; duplicate keys in CSV (last wins): ${r.duplicates.join(', ')}` : '')
|
|
);
|
|
} finally {
|
|
await sql.end();
|
|
}
|