data holders: private tracker of every organisation holding my address, email, phone or IBAN
Some checks failed
Deploy production / deploy (push) Has been cancelled

dataholders schema (orgs + identity) with migration and personal_app grants,
idempotent CSV import (upsert on name+login, never wipes, keeps site edits and
ticks), owner-only /trackers/data-holders with search, filters, pagination,
inline edit/add and a moving checklist per identifier
This commit is contained in:
2026-10-25 16:09:28 +00:00
parent fd4a34b495
commit 772e699655
18 changed files with 30891 additions and 1 deletions

View File

@@ -0,0 +1,48 @@
import { describe, expect, it } from 'vitest';
import { isDue, matchesCurrent, type OrgLike } from './dataholders';
const base: OrgLike = {
archived: false,
action: 'update',
holdsAddress: true,
holdsEmail: true,
holdsPhone: false,
holdsIban: false,
addressOnFile: 'Houtenstraat 15, 2801 WD Gouda',
emailOnFile: 'old@example.com',
phoneOnFile: null,
addressUpdatedOn: null,
emailUpdatedOn: null,
phoneUpdatedOn: null,
ibanUpdatedOn: null
};
const address = {
kind: 'address',
value: 'Van Baerlestraat 12, 2802 GJ Gouda',
since: '2026-10-15'
};
describe('data holders', () => {
it('matches addresses on street and number', () => {
expect(matchesCurrent('Van Baerlestraat 12 (Mar 2025 mail)', address.value, 'address')).toBe(
true
);
expect(matchesCurrent('Houtenstraat 15', address.value, 'address')).toBe(false);
});
it('is due until ticked on or after the change', () => {
expect(isDue(base, 'address', address)).toBe(true);
expect(isDue({ ...base, addressUpdatedOn: '2026-10-01' }, 'address', address)).toBe(true);
expect(isDue({ ...base, addressUpdatedOn: '2026-10-20' }, 'address', address)).toBe(false);
});
it('skips archived, no-action, not-held and unset identifiers', () => {
expect(isDue({ ...base, archived: true }, 'address', address)).toBe(false);
expect(isDue({ ...base, action: 'none' }, 'address', address)).toBe(false);
expect(isDue({ ...base, holdsAddress: false }, 'address', address)).toBe(false);
expect(isDue(base, 'address', undefined)).toBe(false);
});
it('treats an org with the new address on file as done', () => {
expect(isDue({ ...base, addressOnFile: 'Van Baerlestraat 12' }, 'address', address)).toBe(
false
);
});
});

78
src/lib/dataholders.ts Normal file
View File

@@ -0,0 +1,78 @@
// Shared (client + server) bits of the private data-holders tracker: which identifiers we
// track, and when an org still has an old value ("due" in the move checklist).
export const identityKinds = ['address', 'email', 'phone', 'iban'] as const;
export type IdentityKind = (typeof identityKinds)[number];
export const isIdentityKind = (v: unknown): v is IdentityKind =>
typeof v === 'string' && (identityKinds as readonly string[]).includes(v);
export const identityLabels: Record<IdentityKind, string> = {
address: 'Address',
email: 'Email',
phone: 'Phone',
iban: 'IBAN'
};
export const actions = ['update', 'close', 'check', 'none'] as const;
export const priorities = ['must', 'should', 'nice'] as const;
export const addressStatuses = ['confirmed', 'likely', 'unknown'] as const;
export const priorityLabels = { must: 'Must-do', should: 'Should-do', nice: 'Nice-to-have' };
export type Identity = { kind: string; value: string; since: string };
export type OrgLike = {
archived: boolean;
action: string;
holdsAddress: boolean;
holdsEmail: boolean;
holdsPhone: boolean;
holdsIban: boolean;
addressOnFile: string | null;
emailOnFile: string | null;
phoneOnFile: string | null;
addressUpdatedOn: string | null;
emailUpdatedOn: string | null;
phoneUpdatedOn: string | null;
ibanUpdatedOn: string | null;
};
export const holds = (o: OrgLike, kind: IdentityKind) =>
({ address: o.holdsAddress, email: o.holdsEmail, phone: o.holdsPhone, iban: o.holdsIban })[kind];
export const updatedOn = (o: OrgLike, kind: IdentityKind) =>
({
address: o.addressUpdatedOn,
email: o.emailUpdatedOn,
phone: o.phoneUpdatedOn,
iban: o.ibanUpdatedOn
})[kind];
const norm = (s: string) =>
s
.toLowerCase()
.normalize('NFKD')
.replace(/[^a-z0-9@.]+/g, ' ')
.trim();
/** Does the value on file already match the current one? Addresses match on street + number. */
export function matchesCurrent(onFile: string | null, current: string, kind: IdentityKind) {
if (!onFile || !current) return false;
const head = kind === 'address' ? current.split(',')[0] : current;
return norm(onFile).includes(norm(head));
}
/** Still has the old value: holds it, not archived/no-action, not ticked since the change. */
export function isDue(o: OrgLike, kind: IdentityKind, current: Identity | undefined): boolean {
if (!current || o.archived || o.action === 'none' || !holds(o, kind)) return false;
const ticked = updatedOn(o, kind);
if (ticked && ticked >= current.since) return false;
const onFile = {
address: o.addressOnFile,
email: o.emailOnFile,
phone: o.phoneOnFile,
iban: null
}[kind];
if (kind === 'email' && onFile)
return !onFile.split(/[;,]\s*/).every((e) => matchesCurrent(e, current.value, kind));
return !matchesCurrent(onFile, current.value, kind);
}

View File

@@ -176,6 +176,13 @@ const allTrackers: readonly (Tracker & { private: boolean })[] = [
external: false,
private: true
},
{
name: 'holders',
subtitle: 'private: who has my address, email, phone and bank details, and a moving checklist',
href: '/trackers/data-holders',
external: false,
private: true
},
{
name: 'energy',
subtitle: 'Dutch electricity mix, utilization and gas storage',

View File

@@ -0,0 +1,321 @@
// Importer for the private data-holders tracker (scripts/dataholders-import.ts).
// Idempotent and re-runnable: rows are matched on a stable import key (normalised name +
// login), nothing is ever deleted, checklist ticks and archive flags are never touched, and
// rows edited on the site (edited_at set) only get their empty fields filled.
import type { Sql } from 'postgres';
export type CsvRecord = Record<string, string>;
/** RFC 4180 CSV (quoted fields, "" escapes, newlines inside quotes). */
export function parseCsv(text: string): CsvRecord[] {
const rows: string[][] = [];
let row: string[] = [];
let field = '';
let quoted = false;
for (let i = 0; i < text.length; i++) {
const ch = text[i];
if (quoted) {
if (ch === '"' && text[i + 1] === '"') {
field += '"';
i++;
} else if (ch === '"') quoted = false;
else field += ch;
} else if (ch === '"') quoted = true;
else if (ch === ',') {
row.push(field);
field = '';
} else if (ch === '\n' || ch === '\r') {
if (ch === '\r' && text[i + 1] === '\n') i++;
row.push(field);
rows.push(row);
row = [];
field = '';
} else field += ch;
}
if (field || row.length) {
row.push(field);
rows.push(row);
}
const [header, ...body] = rows.filter((r) => r.some((c) => c.trim()));
if (!header) return [];
const keys = header.map((h) =>
h
.trim()
.toLowerCase()
.replace(/^\ufeff/, '')
);
return body.map((r) => Object.fromEntries(keys.map((k, i) => [k, (r[i] ?? '').trim()])));
}
export const normKey = (s: string | null | undefined) =>
(s ?? '')
.toLowerCase()
.normalize('NFKD')
.replace(/[\u0300-\u036f]/g, '')
.replace(/[^a-z0-9@.]+/g, ' ')
.trim();
export const importKey = (name: string, login: string | null) =>
`${normKey(name)}|${normKey(login)}`;
const CATEGORY: Record<string, string> = {
hosting: 'Domains / hosting',
'hosting / domains': 'Domains / hosting',
payroll: 'Employer',
'government / business': 'Business',
'business admin': 'Business'
};
export type OrgRow = {
name: string;
category: string | null;
login: string | null;
source: string | null;
domains: string | null;
holdsAddress: boolean;
holdsEmail: boolean;
holdsPhone: boolean;
holdsIban: boolean;
holdsBsn: boolean;
holdsCard: boolean;
holdsDob: boolean;
holdsIdDoc: boolean;
dataHeld: string | null;
addressOnFile: string | null;
addressStatus: 'confirmed' | 'likely' | 'unknown' | null;
followsBrp: boolean;
emailOnFile: string | null;
phoneOnFile: string | null;
action: 'update' | 'close' | 'check' | 'none' | null;
priority: 'must' | 'should' | 'nice' | null;
howToChange: string | null;
link: string | null;
evidence: string | null;
notes: string | null;
lastChecked: string | null;
/** Only applied to new rows (CSV status "Done"). */
addressUpdatedOn: string | null;
};
const blank = (v: string | undefined) => (v && v.trim() ? v.trim() : null);
const truthy = (v: string | undefined) => /^(true|yes|1|y)$/i.test(v?.trim() ?? '');
const isoDate = (v: string | undefined) =>
v && /^\d{4}-\d{2}-\d{2}$/.test(v.trim()) ? v.trim() : null;
/** Split the free-text "address on file" into the address itself and how sure we are. */
export function parseAddress(raw: string | null): {
address: string | null;
status: OrgRow['addressStatus'];
followsBrp: boolean;
note: string | null;
} {
if (!raw || /^none( held)?$/i.test(raw))
return { address: null, status: null, followsBrp: false, note: null };
if (/^follows brp/i.test(raw))
return { address: null, status: 'unknown', followsBrp: true, note: null };
const paren = raw.match(/\(([^)]*)\)/)?.[1] ?? null;
if (/^unknown/i.test(raw))
return {
address: null,
status: 'unknown',
followsBrp: false,
note: paren ? `Address: ${paren}` : null
};
if (/^still previous owner/i.test(raw))
return { address: null, status: 'confirmed', followsBrp: false, note: `Address: ${raw}` };
let status: OrgRow['addressStatus'] = 'unknown';
let rest = raw;
if (/^likely\s+/i.test(rest)) {
status = 'likely';
rest = rest.replace(/^likely\s+/i, '');
} else if (/seen in mail|\bmail\)/i.test(rest)) status = 'confirmed';
const [main, ...more] = rest.split(';');
let address = main.replace(/\s*\([^)]*\)\s*/g, ' ').trim();
let note = [paren && !/^seen in mail$/i.test(paren) ? paren : null, ...more.map((m) => m.trim())]
.filter(Boolean)
.join('; ');
const forDomain = address.match(/^(.*?)\s+for\s+(\S+)$/);
if (forDomain) {
address = forDomain[1];
note = [note, `for ${forDomain[2]}`].filter(Boolean).join('; ');
}
if (/^old address$/i.test(address)) address = '';
return {
address: address || null,
status,
followsBrp: false,
note: note ? `Address: ${note}` : null
};
}
export function mapCsvRow(r: CsvRecord): OrgRow | null {
const name = blank(r.name);
if (!name) return null;
const held = r.data_held ?? '';
// Placeholders like "(address/postcode stored as login)" are not a real login.
const rawLogin = blank(r.login);
const login = rawLogin && /^\(.*\)$/.test(rawLogin) ? null : rawLogin;
const email = blank(r.email_on_file);
const addr = parseAddress(blank(r.address_on_file));
const cat = blank(r.category);
const action = blank(r.action)?.toLowerCase() ?? null;
const prio = blank(r.priority)?.toLowerCase() ?? '';
const status = blank(r.status)?.toLowerCase() ?? '';
const notes = [blank(r.notes), addr.note].filter(Boolean).join(' ');
return {
name,
category: cat ? (CATEGORY[cat.toLowerCase()] ?? cat) : null,
login,
source: blank(r.source),
domains: blank(r.domains),
holdsAddress: /address|delivery|location|postcode|lease/i.test(held) || !!addr.address,
holdsEmail: /e-?mail/i.test(held) || !!email || !!login?.includes('@'),
holdsPhone: /phone|mobile/i.test(held),
holdsIban: /\biban\b|linked bank/i.test(held),
holdsBsn: /\bbsn\b|tax id/i.test(held),
holdsCard: /\bcard\b/i.test(held),
holdsDob: /date of birth|birth ?date|\bdob\b/i.test(held),
holdsIdDoc: /id document|id check|passport|photo/i.test(held) || /\bID\b/.test(held),
dataHeld: blank(held),
addressOnFile: addr.address,
addressStatus: addr.status,
followsBrp: truthy(r.auto_from_brp) || addr.followsBrp,
emailOnFile: email,
phoneOnFile: blank(r.phone_on_file),
action:
status === 'not needed'
? 'none'
: action && ['update', 'close', 'check', 'none'].includes(action)
? (action as OrgRow['action'])
: null,
priority: prio.startsWith('must')
? 'must'
: prio.startsWith('should')
? 'should'
: prio.startsWith('nice')
? 'nice'
: null,
howToChange: blank(r.how_to_change),
link: blank(r.link),
evidence: blank(r.evidence),
notes: notes || null,
lastChecked: isoDate(r.last_checked),
addressUpdatedOn: status === 'done' ? (isoDate(r.last_checked) ?? null) : null
};
}
type Existing = {
id: number;
import_key: string | null;
name: string;
login: string | null;
edited_at: Date | null;
} & Record<string, unknown>;
const TEXT = [
'category',
'source',
'domains',
'data_held',
'address_on_file',
'email_on_file',
'phone_on_file',
'how_to_change',
'link',
'evidence',
'notes',
'last_checked'
] as const;
const ENUM = ['address_status', 'action', 'priority'] as const;
const FLAGS = [
'holds_address',
'holds_email',
'holds_phone',
'holds_iban',
'holds_bsn',
'holds_card',
'holds_dob',
'holds_id_doc',
'follows_brp'
] as const;
const snake = (k: string) => k.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`);
export async function importOrgs(sql: Sql, rows: OrgRow[], { dryRun = false } = {}) {
const result = {
inserted: 0,
updated: 0,
unchanged: 0,
linkedLogin: 0,
duplicates: [] as string[]
};
const byKey = new Map<string, OrgRow>();
for (const r of rows) {
const key = importKey(r.name, r.login);
if (byKey.has(key)) result.duplicates.push(r.name);
byKey.set(key, r);
}
await sql.begin(async (tx) => {
const existing = await tx<Existing[]>`select * from dataholders.orgs`;
const index = new Map(existing.filter((e) => e.import_key).map((e) => [e.import_key!, e]));
for (const [key, r] of byKey) {
const values = Object.fromEntries(Object.entries(r).map(([k, v]) => [snake(k), v])) as Record<
string,
unknown
>;
let row = index.get(key);
// A login appearing for an org that so far had none: attach it instead of duplicating.
if (!row && r.login) {
const nameKey = importKey(r.name, null);
const bare = existing.filter(
(e) =>
!e.login &&
(e.import_key === nameKey || (!e.import_key && importKey(e.name, null) === nameKey))
);
if (bare.length === 1) {
row = bare[0];
result.linkedLogin++;
}
}
if (!row) {
const insert: Record<string, unknown> = { name: r.name, login: r.login, import_key: key };
for (const c of [...TEXT, ...FLAGS, 'address_updated_on'])
if (values[c] !== null && values[c] !== undefined) insert[c] = values[c];
for (const c of ENUM) if (values[c]) insert[c] = values[c];
if (!dryRun) await tx`insert into dataholders.orgs ${tx(insert)}`;
result.inserted++;
continue;
}
const edited = !!row.edited_at;
const patch: Record<string, unknown> = {};
const set = (c: string, v: unknown) => {
const cur =
row![c] instanceof Date ? (row![c] as Date).toISOString().slice(0, 10) : row![c];
if (v !== cur) patch[c] = v;
};
if (row.import_key !== key) set('import_key', key);
if (!row.login && r.login) set('login', r.login);
if (!edited && r.name !== row.name) set('name', r.name);
for (const c of [...TEXT, ...ENUM]) {
const v = values[c];
if (v === null || v === undefined || v === '') continue; // never wipe
if (!edited || row[c] === null || row[c] === '') set(c, v);
}
for (const c of FLAGS) if (values[c] && !row[c]) set(c, true);
if (!Object.keys(patch).length) {
result.unchanged++;
continue;
}
if (!dryRun) await tx`update dataholders.orgs set ${tx(patch)} where id = ${row.id}`;
result.updated++;
}
});
return result;
}
/** Seed the current identity values once (never overwrites what the site set). */
export async function seedIdentity(
sql: Sql,
values: { kind: string; value: string; since: string }[]
) {
for (const v of values)
await sql`insert into dataholders.identity ${sql(v)} on conflict (kind) do nothing`;
}

View File

@@ -0,0 +1,79 @@
import { boolean, check, date, index, integer, text, uniqueIndex } from 'drizzle-orm/pg-core';
import { sql } from 'drizzle-orm';
import { dataholdersSchema } from './schemas';
import { timestamps, timestamptz } from './helpers';
// Private: every organisation that holds data about Stan (owner only, see requireOwner).
// Whenever the address, email, phone or IBAN changes, set the new value in
// dataholders.identity and work down the orgs that still have the old one (the
// /trackers/data-holders checklist). Never stores passwords: `login` is a username or email.
export const dataholderOrgs = dataholdersSchema.table(
'orgs',
{
id: integer('id').primaryKey().generatedAlwaysAsIdentity(),
name: text('name').notNull(),
category: text('category').notNull().default('Other'),
/** Username or email used to log in. Never a password. */
login: text('login'),
/** Where the row came from, e.g. "Daisy checklist", "1Password". */
source: text('source'),
/** Websites/domains of the account (from 1Password). */
domains: text('domains'),
/** Stable import key (normalised name + login). Set by the importer, never by edits. */
importKey: text('import_key'),
holdsAddress: boolean('holds_address').notNull().default(false),
holdsEmail: boolean('holds_email').notNull().default(false),
holdsPhone: boolean('holds_phone').notNull().default(false),
holdsIban: boolean('holds_iban').notNull().default(false),
holdsBsn: boolean('holds_bsn').notNull().default(false),
holdsCard: boolean('holds_card').notNull().default(false),
holdsDob: boolean('holds_dob').notNull().default(false),
holdsIdDoc: boolean('holds_id_doc').notNull().default(false),
/** Free-text description of everything they hold (the "other" bucket). */
dataHeld: text('data_held'),
/** The address they have, as last seen (often an old one). */
addressOnFile: text('address_on_file'),
addressStatus: text('address_status').notNull().default('unknown'),
/** Gets the address from the BRP automatically (government bodies). */
followsBrp: boolean('follows_brp').notNull().default(false),
emailOnFile: text('email_on_file'),
phoneOnFile: text('phone_on_file'),
action: text('action').notNull().default('update'),
priority: text('priority').notNull().default('should'),
howToChange: text('how_to_change'),
link: text('link'),
evidence: text('evidence'),
notes: text('notes'),
addressUpdatedOn: date('address_updated_on', { mode: 'string' }),
emailUpdatedOn: date('email_updated_on', { mode: 'string' }),
phoneUpdatedOn: date('phone_updated_on', { mode: 'string' }),
ibanUpdatedOn: date('iban_updated_on', { mode: 'string' }),
lastChecked: date('last_checked', { mode: 'string' }),
/** Closed accounts stay in the list (archived), so their history is kept. */
archived: boolean('archived').notNull().default(false),
/** Set on any edit from the site; the importer then only fills empty fields. */
editedAt: timestamptz('edited_at'),
...timestamps
},
(t) => [
uniqueIndex('orgs_import_key_key').on(t.importKey),
index('orgs_category_idx').on(t.category),
check('orgs_name_check', sql`length(btrim(${t.name})) > 0`),
check('orgs_address_status_check', sql`${t.addressStatus} IN ('confirmed','likely','unknown')`),
check('orgs_action_check', sql`${t.action} IN ('update','close','check','none')`),
check('orgs_priority_check', sql`${t.priority} IN ('must','should','nice')`)
]
);
/** The current value of each identifier, and since when. Orgs ticked before `since` are due. */
export const dataholderIdentity = dataholdersSchema.table(
'identity',
{
kind: text('kind').primaryKey(),
value: text('value').notNull(),
since: date('since', { mode: 'string' }).notNull(),
...timestamps
},
(t) => [check('identity_kind_check', sql`${t.kind} IN ('address','email','phone','iban')`)]
);

View File

@@ -25,3 +25,4 @@ export * from './wildlife';
export * from './trains';
export * from './music';
export * from './hardware';
export * from './dataholders';

View File

@@ -23,3 +23,4 @@ export const hardwareSchema = pgSchema('hardware');
export const planesSchema = pgSchema('planes');
export const wildlifeSchema = pgSchema('wildlife');
export const trainsSchema = pgSchema('trains');
export const dataholdersSchema = pgSchema('dataholders');

View File

@@ -21,6 +21,8 @@ describe('owner gate', () => {
expect(isOwnerWritePath('/trackers/tasks')).toBe(true);
expect(isOwnerOnlyPath('/trackers/groceries')).toBe(true);
expect(isOwnerWritePath('/trackers/groceries')).toBe(true);
expect(isOwnerOnlyPath('/trackers/data-holders')).toBe(true);
expect(isOwnerWritePath('/trackers/data-holders')).toBe(true);
expect(isOwnerOnlyPath('/trackers/houses')).toBe(true);
expect(isOwnerWritePath('/trackers/houses')).toBe(true);
});

View File

@@ -25,6 +25,7 @@ export const ownerOnlyPrefixes = [
'/trackers/tasks',
'/trackers/groceries',
'/trackers/houses',
'/trackers/data-holders',
'/trackers/books/files',
'/lists/db',
// Spotify OAuth for the listening tracker (connect + callback 404 for everyone else).
@@ -47,6 +48,7 @@ export const ownerWritePrefixes = [
'/trackers/tasks',
'/trackers/groceries',
'/trackers/houses',
'/trackers/data-holders',
'/trackers/books',
'/trackers/cars',
'/trackers/planes',

View File

@@ -84,6 +84,18 @@ export const trackerIcons: Record<string, TrackerIcon> = {
ink: WHITE,
paths: ['M3 10.5 12 3l9 7.5', 'M5 9.5V20h14V9.5', 'M10 20v-6h4v6']
},
holders: {
from: '#38bdf8',
to: '#6d28d9',
ink: WHITE,
// id card
paths: [
'M4 5h16a2 2 0 0 1 2 2v10a2 2 0 0 1-2 2H4a2 2 0 0 1-2-2V7a2 2 0 0 1 2-2z',
c(9, 10.5, 2),
'M5.8 16a3.4 3.4 0 0 1 6.4 0',
'M15 10h4M15 14h3'
]
},
energy: {
from: '#fde047',
to: '#eab308',