data holders: private tracker of every organisation holding my address, email, phone or IBAN
Some checks failed
Deploy production / deploy (push) Has been cancelled
Some checks failed
Deploy production / deploy (push) Has been cancelled
dataholders schema (orgs + identity) with migration and personal_app grants, idempotent CSV import (upsert on name+login, never wipes, keeps site edits and ticks), owner-only /trackers/data-holders with search, filters, pagination, inline edit/add and a moving checklist per identifier
This commit is contained in:
61
scripts/dataholders-import.ts
Normal file
61
scripts/dataholders-import.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Import organisations that hold data about Stan into the private `dataholders` schema.
|
||||
* Idempotent: re-run it whenever the CSV grows. Never deletes rows, never touches checklist
|
||||
* ticks or archive flags, and only fills empty fields on rows edited on the site.
|
||||
*
|
||||
* bun scripts/dataholders-import.ts data.csv # import
|
||||
* bun scripts/dataholders-import.ts --dry-run data.csv # validate and show what would change
|
||||
* bun scripts/dataholders-import.ts --address "Street 1, 1234 AB City" --since 2026-10-15 data.csv
|
||||
*
|
||||
* Columns (header names, any order): name, category, data_held, address_on_file,
|
||||
* email_on_file, phone_on_file, action, how_to_change, link, auto_from_brp, priority, status,
|
||||
* evidence, notes, last_checked, source, login. A `password` column is refused.
|
||||
* Uses DB_URL (the app role on prod, personal_dev on dev).
|
||||
*/
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import postgres from 'postgres';
|
||||
import {
|
||||
importOrgs,
|
||||
mapCsvRow,
|
||||
parseCsv,
|
||||
seedIdentity
|
||||
} from '../src/lib/server/dataholders-import';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const opt = (name: string) => {
|
||||
const i = args.indexOf(name);
|
||||
return i >= 0 ? args.splice(i, 2)[1] : undefined;
|
||||
};
|
||||
const address = opt('--address');
|
||||
const since = opt('--since');
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const files = args.filter((a) => !a.startsWith('--'));
|
||||
if (files.length !== 1) {
|
||||
console.error(
|
||||
'usage: bun scripts/dataholders-import.ts [--dry-run] [--address A --since YYYY-MM-DD] <orgs.csv>'
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
const records = parseCsv(await readFile(files[0], 'utf8'));
|
||||
if (records.length && Object.keys(records[0]).some((k) => /pass(word)?|secret|otp|totp/i.test(k))) {
|
||||
console.error('Refusing: the CSV has a password/secret column. Remove it first.');
|
||||
process.exit(1);
|
||||
}
|
||||
const rows = records.map(mapCsvRow).filter((r) => r !== null);
|
||||
console.log(`Valid: ${rows.length} of ${records.length} rows`);
|
||||
if (!process.env.DB_URL) throw new Error('DB_URL is required');
|
||||
const sql = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
|
||||
try {
|
||||
if (address && !dryRun)
|
||||
await seedIdentity(sql, [
|
||||
{ kind: 'address', value: address, since: since ?? new Date().toISOString().slice(0, 10) }
|
||||
]);
|
||||
const r = await importOrgs(sql, rows, { dryRun });
|
||||
console.log(
|
||||
`${dryRun ? '[dry run] ' : ''}${r.inserted} inserted, ${r.updated} updated, ${r.unchanged} unchanged, ` +
|
||||
`${r.linkedLogin} logins attached to existing orgs` +
|
||||
(r.duplicates.length ? `; duplicate keys in CSV (last wins): ${r.duplicates.join(', ')}` : '')
|
||||
);
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
Reference in New Issue
Block a user