From 941a9f16e192026bd41682c7eb28cc78bf4f7749 Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sat, 24 Oct 2026 02:42:31 +0000 Subject: [PATCH] data browser wip --- src/lib/components/data/FieldInput.svelte | 146 ++++ src/lib/components/data/data-format.ts | 72 ++ src/lib/server/data-editor.ts | 675 ++++++++++++++++++ src/lib/server/data-policy.test.ts | 126 ++++ src/lib/server/data-policy.ts | 217 ++++++ src/lib/server/data-route.ts | 47 ++ src/lib/server/lists.ts | 2 + src/lib/server/owner.ts | 3 +- src/lib/server/slugify.ts | 7 +- src/routes/lists/+page.server.ts | 13 +- src/routes/lists/+page.svelte | 65 ++ src/routes/lists/[slug]/+page.server.ts | 4 +- src/routes/lists/[slug]/+page.svelte | 4 + src/routes/lists/db/+layout.server.ts | 10 + src/routes/lists/db/+page.server.ts | 9 + .../lists/db/[schema]/[table]/+page.server.ts | 19 + .../lists/db/[schema]/[table]/+page.svelte | 585 +++++++++++++++ .../db/[schema]/[table]/columns/+server.ts | 25 + .../db/[schema]/[table]/export/+server.ts | 23 + .../db/[schema]/[table]/import/+server.ts | 26 + .../db/[schema]/[table]/lookup/+server.ts | 13 + .../lists/db/[schema]/[table]/rows/+server.ts | 56 ++ 22 files changed, 2141 insertions(+), 6 deletions(-) create mode 100644 src/lib/components/data/FieldInput.svelte create mode 100644 src/lib/components/data/data-format.ts create mode 100644 src/lib/server/data-editor.ts create mode 100644 src/lib/server/data-policy.test.ts create mode 100644 src/lib/server/data-policy.ts create mode 100644 src/lib/server/data-route.ts create mode 100644 src/routes/lists/db/+layout.server.ts create mode 100644 src/routes/lists/db/+page.server.ts create mode 100644 src/routes/lists/db/[schema]/[table]/+page.server.ts create mode 100644 src/routes/lists/db/[schema]/[table]/+page.svelte create mode 100644 src/routes/lists/db/[schema]/[table]/columns/+server.ts create mode 100644 src/routes/lists/db/[schema]/[table]/export/+server.ts create mode 100644 src/routes/lists/db/[schema]/[table]/import/+server.ts create mode 100644 src/routes/lists/db/[schema]/[table]/lookup/+server.ts create mode 100644 src/routes/lists/db/[schema]/[table]/rows/+server.ts diff --git a/src/lib/components/data/FieldInput.svelte b/src/lib/components/data/FieldInput.svelte new file mode 100644 index 0000000..b8b1a5b --- /dev/null +++ b/src/lib/components/data/FieldInput.svelte @@ -0,0 +1,146 @@ + + +
+ {#if kind === 'bool'} + + {:else if kind === 'enum' || (column.uiType === 'select' && column.enumValues.length)} + + {:else if long} + + {:else} + { + value = e.currentTarget.value; + if (column.fk) queueLookup(e.currentTarget.value); + }} + onkeydown={keydown} + {@attach focusOnMount} + aria-label={column.label} + /> + {#if column.fk} + + {#each options as o (o.value)}{/each} + + {/if} + {/if} + {#if column.nullable && value !== null} + + {/if} +
diff --git a/src/lib/components/data/data-format.ts b/src/lib/components/data/data-format.ts new file mode 100644 index 0000000..fe00fcc --- /dev/null +++ b/src/lib/components/data/data-format.ts @@ -0,0 +1,72 @@ +// Client-side value helpers for the /lists/db grid (shared by the grid and FieldInput). +export type Col = { + name: string; + label: string; + type: string; + kind: string; + uiType: string | null; + nullable: boolean; + hasDefault: boolean; + defaultExpr: string | null; + readOnly: boolean; + enumValues: string[]; + pk: boolean; + fk: { schema: string; table: string; column: string } | null; + help: string | null; +}; + +const pad = (n: number) => String(n).padStart(2, '0'); +const withTz = (c: Col) => c.type.includes('with time zone'); + +/** Raw JSON value from the server -> editor string (null stays null). */ +export function toEditor(c: Col, v: unknown): string | null { + if (v === null || v === undefined) return null; + if (c.kind === 'json') return JSON.stringify(v, null, 2); + if (c.kind === 'array') return JSON.stringify(v); + if (c.kind === 'datetime' && typeof v === 'string') { + if (!withTz(c)) return v.slice(0, 19); + const d = new Date(v); + if (Number.isNaN(d.getTime())) return v; + return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}T${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`; + } + if (c.kind === 'date' && typeof v === 'string') return v.slice(0, 10); + return String(v); +} + +/** Editor string -> value sent to the server. */ +export function fromEditor(c: Col, s: string | null): string | null { + if (s === null) return null; + if (c.kind === 'datetime' && withTz(c) && s.trim()) { + const d = new Date(s); + return Number.isNaN(d.getTime()) ? s : d.toISOString(); + } + return s; +} + +/** Short display text for a grid cell. */ +export function display(c: Col, v: unknown): string { + if (v === null || v === undefined) return ''; + if (c.kind === 'bool') return v ? '✓' : '✗'; + if (c.uiType === 'cents' && typeof v === 'number') return `€${(v / 100).toFixed(2)}`; + if (c.kind === 'datetime' && typeof v === 'string') { + const d = new Date(v); + if (Number.isNaN(d.getTime())) return v; + return withTz(c) + ? d.toLocaleString('en-GB', { dateStyle: 'medium', timeStyle: 'short' }) + : v.replace('T', ' ').slice(0, 16); + } + if (typeof v === 'object') { + const s = JSON.stringify(v); + return s.length > 120 ? `${s.slice(0, 117)}…` : s; + } + const s = String(v); + return s.length > 160 ? `${s.slice(0, 157)}…` : s; +} + +export function shortType(t: string): string { + return t + .replace('timestamp with time zone', 'timestamptz') + .replace('timestamp without time zone', 'timestamp') + .replace('character varying', 'varchar') + .replace('double precision', 'float8'); +} diff --git a/src/lib/server/data-editor.ts b/src/lib/server/data-editor.ts new file mode 100644 index 0000000..10a40de --- /dev/null +++ b/src/lib/server/data-editor.ts @@ -0,0 +1,675 @@ +// Owner-only database browser/editor behind /lists/db. Every table and column name used +// in SQL comes from pg_catalog (looked up with bound parameters) and is quoted with +// sql.identifier; every value is a bound parameter. Access rules live in data-policy.ts. +import { sql, type SQL } from 'drizzle-orm'; +import { db } from '#lib/server/db/index.js'; +import { + coerce, + hiddenColumn, + plausibleIdent, + safeTypeName, + tableAccess, + typeKind, + writeAllowed, + type Access, + type Filter, + type Kind +} from '#lib/server/data-policy.js'; + +export type TableSummary = { + schema: string; + name: string; + displayName: string; + description: string | null; + relkind: string; + access: Access; + rows: number | null; + approximate: boolean; + listSlug: string | null; +}; + +export type ColumnInfo = { + name: string; + label: string; + type: string; + kind: Kind; + uiType: string | null; + nullable: boolean; + hasDefault: boolean; + defaultExpr: string | null; + /** identity ALWAYS / generated columns cannot be written */ + readOnly: boolean; + enumValues: string[]; + pk: boolean; + fk: { schema: string; table: string; column: string } | null; + help: string | null; +}; + +export type TableInfo = TableSummary & { + oid: number; + columns: ColumnInfo[]; + primaryKey: string[]; + canInsert: boolean; + canUpdate: boolean; + canDelete: boolean; + isList: boolean; +}; + +type CatalogRow = { + oid: number; + schema: string; + name: string; + relkind: string; + estimate: number; + can_insert: boolean; + can_update: boolean; + can_delete: boolean; + has_pk: boolean; + comment: string | null; + display_name: string | null; + description: string | null; + meta_kind: string | null; + slug: string | null; + archived: boolean; +}; + +const MAX_EXACT = 200_000; + +function catalogQuery(where: SQL) { + return sql` + SELECT c.oid::int AS oid, n.nspname AS schema, c.relname AS name, c.relkind::text AS relkind, + greatest(c.reltuples, 0)::bigint::float8 AS estimate, + has_table_privilege(c.oid, 'INSERT') AS can_insert, + has_table_privilege(c.oid, 'UPDATE') AS can_update, + has_table_privilege(c.oid, 'DELETE') AS can_delete, + EXISTS (SELECT 1 FROM pg_index i WHERE i.indrelid = c.oid AND i.indisprimary) AS has_pk, + obj_description(c.oid, 'pg_class') AS comment, + mt.display_name, mt.description, mt.kind AS meta_kind, mt.slug, + (mt.archived_at IS NOT NULL) AS archived + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN meta.tables mt ON mt.schema_name = n.nspname AND mt.table_name = c.relname + WHERE c.relkind IN ('r', 'p', 'v', 'm') AND NOT c.relispartition + AND n.nspname <> 'information_schema' AND n.nspname NOT LIKE 'pg\\_%' + AND has_schema_privilege(n.oid, 'USAGE') AND has_table_privilege(c.oid, 'SELECT') + AND ${where} + ORDER BY n.nspname, c.relname`; +} + +function summarise(r: CatalogRow): Omit { + const access = tableAccess({ + schema: r.schema, + name: r.name, + relkind: r.relkind, + hasPrimaryKey: r.has_pk, + canInsert: r.can_insert, + canUpdate: r.can_update, + canDelete: r.can_delete + }); + return { + schema: r.schema, + name: r.name, + displayName: r.display_name ?? r.name.replace(/_/g, ' '), + description: r.description ?? r.comment, + relkind: r.relkind, + access: r.archived ? 'locked' : access, + listSlug: r.meta_kind === 'list' && !r.archived ? r.slug : null + }; +} + +const rel = (schema: string, name: string) => sql`${sql.identifier(schema)}.${sql.identifier(name)}`; + +/** Every table/view the app role can read, minus hidden ones, with row counts. */ +export async function listTables(): Promise { + const rows = (await db.execute(catalogQuery(sql`true`))) as unknown as CatalogRow[]; + const visible = rows.map((r) => ({ r, s: summarise(r) })).filter(({ s }) => s.access !== 'hidden'); + // exact counts for real tables up to MAX_EXACT rows; estimates for huge ones; views are + // counted only when opened (some aggregate big tables). + const exact = visible.filter(({ r }) => (r.relkind === 'r' || r.relkind === 'p') && r.estimate < MAX_EXACT); + const counts = new Map(); + if (exact.length) { + const res = await db.transaction(async (tx) => { + await tx.execute(sql`SET LOCAL statement_timeout = '10s'`); + return tx.execute<{ k: string; n: number }>( + sql.join( + exact.map( + ({ r }) => sql`SELECT ${`${r.schema}.${r.name}`}::text AS k, count(*)::int AS n FROM ${rel(r.schema, r.name)}` + ), + sql` UNION ALL ` + ) + ); + }); + for (const c of res as unknown as { k: string; n: number }[]) counts.set(c.k, Number(c.n)); + } + return visible.map(({ r, s }) => { + const n = counts.get(`${r.schema}.${r.name}`); + return n !== undefined + ? { ...s, rows: n, approximate: false } + : r.relkind === 'r' || r.relkind === 'p' + ? { ...s, rows: r.estimate, approximate: true } + : { ...s, rows: null, approximate: false }; + }); +} + +type ColumnRow = { + name: string; + type: string; + notnull: boolean; + default_expr: string | null; + identity: string; + generated: string; + is_enum: boolean; + enum_values: string[] | null; + comment: string | null; + pk_pos: number | null; + fk_schema: string | null; + fk_table: string | null; + fk_column: string | null; + ui_type: string | null; + label: string | null; + help_text: string | null; + meta_hidden: boolean | null; + select_values: string[] | null; +}; + +/** One table by schema + name (bound parameters), or null when unknown or hidden. */ +export async function getTable(schema: string, name: string): Promise { + if (!plausibleIdent(schema) || !plausibleIdent(name)) return null; + const [r] = (await db.execute( + catalogQuery(sql`n.nspname = ${schema} AND c.relname = ${name}`) + )) as unknown as CatalogRow[]; + if (!r) return null; + const s = summarise(r); + if (s.access === 'hidden') return null; + const cols = (await db.execute(sql` + SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS type, a.attnotnull AS notnull, + pg_get_expr(d.adbin, d.adrelid) AS default_expr, a.attidentity::text AS identity, + a.attgenerated::text AS generated, + (et.typtype = 'e') AS is_enum, + (SELECT array_agg(e.enumlabel::text ORDER BY e.enumsortorder) FROM pg_enum e WHERE e.enumtypid = et.oid) AS enum_values, + col_description(a.attrelid, a.attnum) AS comment, + (SELECT array_position(i.indkey::int2[], a.attnum) FROM pg_index i + WHERE i.indrelid = a.attrelid AND i.indisprimary) AS pk_pos, + fk.fk_schema, fk.fk_table, fk.fk_column, + mc.ui_type, mc.label, mc.help_text, mc.hidden AS meta_hidden, + CASE WHEN jsonb_typeof(mc.options->'values') = 'array' + THEN ARRAY(SELECT jsonb_array_elements_text(mc.options->'values')) END AS select_values + FROM pg_attribute a + JOIN pg_type t ON t.oid = a.atttypid + LEFT JOIN pg_type et ON et.oid = CASE WHEN t.typelem <> 0 AND t.typcategory = 'A' THEN t.typelem ELSE t.oid END + LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + LEFT JOIN LATERAL ( + SELECT fn.nspname AS fk_schema, fc.relname AS fk_table, fa.attname AS fk_column + FROM pg_constraint k + JOIN pg_class fc ON fc.oid = k.confrelid + JOIN pg_namespace fn ON fn.oid = fc.relnamespace + JOIN pg_attribute fa ON fa.attrelid = k.confrelid AND fa.attnum = k.confkey[1] + WHERE k.conrelid = a.attrelid AND k.contype = 'f' AND cardinality(k.conkey) = 1 + AND k.conkey[1] = a.attnum + LIMIT 1) fk ON true + LEFT JOIN meta.tables mt ON mt.schema_name = ${schema} AND mt.table_name = ${name} + LEFT JOIN meta.columns mc ON mc.table_id = mt.id AND mc.column_name = a.attname + WHERE a.attrelid = ${r.oid} AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum`)) as unknown as ColumnRow[]; + const columns: ColumnInfo[] = cols + .filter((c) => !hiddenColumn(c.name) && safeTypeName(c.type)) + .map((c) => ({ + name: c.name, + label: c.label ?? c.name.replace(/_/g, ' '), + type: c.type, + kind: typeKind(c.type, Boolean(c.is_enum)), + uiType: c.ui_type, + nullable: !c.notnull, + hasDefault: c.default_expr !== null || c.identity !== '' || c.generated !== '', + defaultExpr: c.default_expr, + readOnly: c.identity === 'a' || c.generated !== '', + enumValues: c.enum_values ?? (c.ui_type === 'select' ? (c.select_values ?? []) : []), + pk: c.pk_pos !== null, + fk: c.fk_schema ? { schema: c.fk_schema, table: c.fk_table!, column: c.fk_column! } : null, + help: c.help_text ?? c.comment + })); + const primaryKey = cols + .filter((c) => c.pk_pos !== null) + .sort((a, b) => a.pk_pos! - b.pk_pos!) + .map((c) => c.name); + // a key column that is hidden or has an odd type makes rows unaddressable: read-only + const keyUsable = primaryKey.length > 0 && primaryKey.every((k) => columns.some((c) => c.name === k)); + return { + ...s, + access: s.access !== 'locked' && !keyUsable ? 'locked' : s.access, + rows: null, + approximate: false, + oid: r.oid, + columns, + primaryKey, + canInsert: r.can_insert, + canUpdate: r.can_update, + canDelete: r.can_delete, + isList: s.listSlug !== null + }; +} + +// ---------- reading ---------- + +export type RowQuery = { + q?: string; + filters?: Filter[]; + sort?: string; + dir?: 'asc' | 'desc'; + limit?: number; + offset?: number; +}; + +function colRef(t: TableInfo, name: string) { + const c = t.columns.find((x) => x.name === name); + if (!c) throw new DataError(400, `Unknown column ${name}.`); + return { c, ref: sql`t.${sql.identifier(c.name)}` }; +} + +function cast(value: string | null, c: ColumnInfo) { + return sql`CAST(${value} AS ${sql.raw(c.type)})`; +} + +function whereClause(t: TableInfo, q: RowQuery): SQL { + const parts: SQL[] = []; + const search = q.q?.trim(); + if (search) { + const texts = t.columns.map((c) => sql`t.${sql.identifier(c.name)}::text`); + parts.push(sql`concat_ws(' ', ${sql.join(texts, sql`, `)}) ILIKE ${`%${search.replace(/[\\%_]/g, '\\$&')}%`}`); + } + for (const f of q.filters ?? []) { + const { c, ref } = colRef(t, f.column); + if (f.op === 'null') parts.push(sql`${ref} IS NULL`); + else if (f.op === 'notnull') parts.push(sql`${ref} IS NOT NULL`); + else if (f.op === 'contains') + parts.push(sql`${ref}::text ILIKE ${`%${f.value.replace(/[\\%_]/g, '\\$&')}%`}`); + else { + const v = coerce(c.kind, f.value, c.label); + if (!v.ok) throw new DataError(400, v.error); + const val = c.kind === 'json' ? sql`${v.value}::jsonb` : cast(v.value, c); + const lhs = c.kind === 'json' ? sql`${ref}::jsonb` : ref; + const op = { eq: '=', ne: 'IS DISTINCT FROM', gt: '>', gte: '>=', lt: '<', lte: '<=' }[f.op]; + parts.push(sql`${lhs} ${sql.raw(op)} ${val}`); + } + } + return parts.length ? sql`WHERE ${sql.join(parts, sql` AND `)}` : sql``; +} + +function orderClause(t: TableInfo, q: RowQuery): SQL { + const keys: SQL[] = []; + if (q.sort) { + const { ref } = colRef(t, q.sort); + keys.push(sql`${ref} ${sql.raw(q.dir === 'desc' ? 'DESC NULLS LAST' : 'ASC NULLS FIRST')}`); + } + for (const k of t.primaryKey.length ? t.primaryKey : []) + if (k !== q.sort) keys.push(sql`t.${sql.identifier(k)} ${sql.raw(q.dir === 'desc' && !q.sort ? 'DESC' : 'ASC')}`); + return keys.length ? sql`ORDER BY ${sql.join(keys, sql`, `)}` : sql``; +} + +/** to_jsonb of the visible columns only (hidden credential columns are never selected). */ +function rowJson(t: TableInfo) { + // jsonb_build_object takes at most 100 arguments: build it in chunks of 50 columns + const chunks: SQL[] = []; + for (let i = 0; i < t.columns.length; i += 50) + chunks.push( + sql`jsonb_build_object(${sql.join( + t.columns.slice(i, i + 50).map((c) => sql`${c.name}::text, t.${sql.identifier(c.name)}`), + sql`, ` + )})` + ); + return chunks.length ? sql.join(chunks, sql` || `) : sql`'{}'::jsonb`; +} + +export type RowsResult = { + rows: Record[]; + total: number | null; + approximate: boolean; + fkLabels: Record>; +}; + +export async function readRows(t: TableInfo, q: RowQuery): Promise { + const limit = Math.min(Math.max(q.limit ?? 50, 1), 500); + const offset = Math.max(q.offset ?? 0, 0); + const where = whereClause(t, q); + const filtered = Boolean(q.q?.trim() || q.filters?.length); + return db.transaction(async (tx) => { + await tx.execute(sql`SET TRANSACTION READ ONLY`); + await tx.execute(sql`SET LOCAL statement_timeout = '15s'`); + const select = rowJson(t); + const rows = (await tx.execute<{ r: Record }>( + sql`SELECT ${select} AS r FROM ${rel(t.schema, t.name)} t ${where} ${orderClause(t, q)} LIMIT ${limit} OFFSET ${offset}` + )) as unknown as { r: Record }[]; + let total: number | null = null; + let approximate = false; + const [est] = (await tx.execute<{ e: number }>( + sql`SELECT greatest(reltuples, 0)::float8 AS e FROM pg_class WHERE oid = ${t.oid}` + )) as unknown as { e: number }[]; + if (!filtered && (t.relkind === 'r' || t.relkind === 'p') && est && est.e > MAX_EXACT) { + total = Math.round(est.e); + approximate = true; + } else { + await tx.execute(sql`SAVEPOINT c`); + try { + await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); + const [c] = (await tx.execute<{ n: number }>( + sql`SELECT count(*)::float8 AS n FROM ${rel(t.schema, t.name)} t ${where}` + )) as unknown as { n: number }[]; + total = Number(c.n); + } catch { + await tx.execute(sql`ROLLBACK TO SAVEPOINT c`); + } + } + const visible = rows.map(({ r }) => { + const out: Record = {}; + for (const c of t.columns) out[c.name] = r[c.name] ?? null; + return out; + }); + const fkLabels: RowsResult['fkLabels'] = {}; + for (const c of t.columns) { + if (!c.fk) continue; + const values = [...new Set(visible.map((r) => r[c.name]).filter((v) => v !== null))].map(String); + if (!values.length) continue; + const labels = await fkOptions(c, { values }, tx).catch(() => []); + if (labels.length) fkLabels[c.name] = Object.fromEntries(labels.map((o) => [o.value, o.label])); + } + return { rows: visible, total, approximate, fkLabels }; + }); +} + +type Tx = Parameters[0]>[0]; +type Exec = Pick; + +/** Rows of the table a foreign key points at, as {value, label} for a dropdown. */ +export async function fkOptions( + c: ColumnInfo, + opts: { q?: string; values?: string[] }, + runner: Exec = db +): Promise<{ value: string; label: string }[]> { + if (!c.fk) return []; + const target = await getTable(c.fk.schema, c.fk.table); + if (!target) return []; + const key = target.columns.find((x) => x.name === c.fk!.column); + if (!key) return []; + const labelCol = + ['name', 'display_name', 'title', 'label', 'slug', 'email', 'code'].map((n) => + target.columns.find((x) => x.name === n) + ).find(Boolean) ?? key; + const k = sql`t.${sql.identifier(key.name)}`; + const l = sql`t.${sql.identifier(labelCol.name)}`; + const conds: SQL[] = []; + if (opts.values?.length) + conds.push(sql`${k}::text IN (${sql.join(opts.values.slice(0, 500).map((v) => sql`${v}`), sql`, `)})`); + if (opts.q?.trim()) { + const like = `%${opts.q.trim().replace(/[\\%_]/g, '\\$&')}%`; + conds.push(sql`(${l}::text ILIKE ${like} OR ${k}::text ILIKE ${like})`); + } + const rows = (await runner.execute<{ value: string; label: string }>(sql` + SELECT ${k}::text AS value, ${l}::text AS label FROM ${rel(target.schema, target.name)} t + ${conds.length ? sql`WHERE ${sql.join(conds, sql` AND `)}` : sql``} + ORDER BY 2 NULLS LAST LIMIT ${opts.values?.length ? 500 : 50}`)) as unknown as { + value: string; + label: string; + }[]; + return rows.map((r) => ({ + value: r.value, + label: labelCol === key ? r.value : `${r.label ?? '—'} (${r.value})` + })); +} + +/** All matching rows for CSV export (capped). */ +export async function exportRows(t: TableInfo, q: RowQuery, cap = 200_000) { + return db.transaction(async (tx) => { + await tx.execute(sql`SET TRANSACTION READ ONLY`); + await tx.execute(sql`SET LOCAL statement_timeout = '60s'`); + const cols = t.columns.map((c) => + c.kind === 'json' || c.kind === 'array' + ? sql`to_json(t.${sql.identifier(c.name)})::text` + : sql`t.${sql.identifier(c.name)}::text` + ); + const rows = (await tx.execute<{ v: (string | null)[] }>( + sql`SELECT ARRAY[${sql.join(cols, sql`, `)}]::text[] AS v FROM ${rel(t.schema, t.name)} t ${whereClause(t, q)} ${orderClause(t, q)} LIMIT ${cap}` + )) as unknown as { v: (string | null)[] }[]; + return rows.map((r) => r.v); + }); +} + +// ---------- writing ---------- + +export class DataError extends Error { + constructor( + public status: number, + message: string + ) { + super(message); + } +} + +export type Actor = { id: string; email: string }; +export type Pk = Record; + +function assertWritable(t: TableInfo, unlocked: boolean, need: 'insert' | 'update' | 'delete') { + if (t.access === 'locked') throw new DataError(403, 'This table is read-only here.'); + if (!writeAllowed(t.access, unlocked)) + throw new DataError(423, 'This is an app table: unlock editing first.'); + const ok = need === 'insert' ? t.canInsert : need === 'update' ? t.canUpdate : t.canDelete; + if (!ok) throw new DataError(403, `The app has no ${need} permission on this table.`); +} + +function assignments(t: TableInfo, values: Record) { + const out: { c: ColumnInfo; v: string | null }[] = []; + for (const [name, raw] of Object.entries(values)) { + const c = t.columns.find((x) => x.name === name); + if (!c) throw new DataError(400, `Unknown column ${name}.`); + if (c.readOnly) throw new DataError(400, `${c.label} is generated and cannot be set.`); + const v = coerce(c.kind, raw, c.label); + if (!v.ok) throw new DataError(400, v.error); + if (v.value === null && !c.nullable && !(c.hasDefault && raw === undefined)) + throw new DataError(400, `${c.label} is required.`); + if (c.kind === 'enum' && v.value !== null && c.enumValues.length && !c.enumValues.includes(v.value)) + throw new DataError(400, `${c.label} must be one of ${c.enumValues.join(', ')}.`); + out.push({ c, v: v.value }); + } + return out; +} + +function pkWhere(t: TableInfo, pk: Pk) { + if (!t.primaryKey.length) throw new DataError(400, 'This table has no primary key.'); + return sql.join( + t.primaryKey.map((k) => { + const c = t.columns.find((x) => x.name === k)!; + const raw = pk[k]; + if (raw === undefined || raw === null) throw new DataError(400, `Missing key ${k}.`); + const v = coerce(c.kind, typeof raw === 'object' ? JSON.stringify(raw) : String(raw), c.label); + if (!v.ok || v.value === null) throw new DataError(400, `Invalid key ${k}.`); + return sql`t.${sql.identifier(k)} = ${cast(v.value, c)}`; + }), + sql` AND ` + ); +} + +let auditAvailable: boolean | null = null; +async function audit( + tx: Exec, + actor: Actor, + t: TableInfo, + action: 'insert' | 'update' | 'delete' | 'import' | 'add_column', + pk: unknown, + before: unknown, + after: unknown +) { + if (auditAvailable === null) { + const [r] = (await tx.execute<{ ok: boolean }>( + sql`SELECT to_regclass('meta.edit_log') IS NOT NULL AND has_table_privilege('meta.edit_log', 'INSERT') AS ok` + )) as unknown as { ok: boolean }[]; + auditAvailable = Boolean(r?.ok); + } + if (!auditAvailable) return; + await tx.execute(sql` + INSERT INTO meta.edit_log (user_id, user_email, schema_name, table_name, action, pk, before, after) + VALUES (${actor.id}, ${actor.email}, ${t.schema}, ${t.name}, ${action}, + ${pk === null ? null : JSON.stringify(pk)}::jsonb, + ${before === null ? null : JSON.stringify(before)}::jsonb, + ${after === null ? null : JSON.stringify(after)}::jsonb)`); +} + +function pgMessage(e: unknown): DataError { + if (e instanceof DataError) return e; + const err = (e as { cause?: unknown })?.cause ?? e; + const code = (err as { code?: string })?.code ?? ''; + const msg = (err as { message?: string })?.message ?? 'Database error'; + // constraint / data errors are the user's input; show Postgres' explanation + if (/^(22|23)/.test(code)) return new DataError(400, msg); + if (code === '42501') return new DataError(403, 'The app has no permission for this change.'); + console.error('data editor write failed', e); + return new DataError(500, 'The database refused the change.'); +} + +async function write(fn: (tx: Tx) => Promise): Promise { + try { + return await db.transaction(async (tx) => { + await tx.execute(sql`SET LOCAL statement_timeout = '30s'`); + return fn(tx); + }); + } catch (e) { + throw pgMessage(e); + } +} + +const one = (rows: unknown) => (rows as T[])[0]; + +export async function insertRow(t: TableInfo, values: Record, unlocked: boolean, actor: Actor) { + assertWritable(t, unlocked, 'insert'); + const set = assignments(t, values); + return write(async (tx) => { + const row = one<{ r: Record }>( + await tx.execute( + set.length + ? sql`INSERT INTO ${rel(t.schema, t.name)} AS t (${sql.join(set.map((s) => sql.identifier(s.c.name)), sql`, `)}) + VALUES (${sql.join(set.map((s) => cast(s.v, s.c)), sql`, `)}) RETURNING ${rowJson(t)} AS r` + : sql`INSERT INTO ${rel(t.schema, t.name)} AS t DEFAULT VALUES RETURNING ${rowJson(t)} AS r` + ) + ).r; + const pk = Object.fromEntries(t.primaryKey.map((k) => [k, row[k]])); + await audit(tx, actor, t, 'insert', pk, null, row); + return row; + }); +} + +export async function updateRow( + t: TableInfo, + pk: Pk, + values: Record, + unlocked: boolean, + actor: Actor +) { + assertWritable(t, unlocked, 'update'); + const set = assignments(t, values); + if (!set.length) throw new DataError(400, 'Nothing to change.'); + const where = pkWhere(t, pk); + return write(async (tx) => { + const before = one<{ r: Record }>( + await tx.execute(sql`SELECT ${rowJson(t)} AS r FROM ${rel(t.schema, t.name)} t WHERE ${where} FOR UPDATE`) + )?.r; + if (!before) throw new DataError(404, 'That row no longer exists.'); + const after = one<{ r: Record }>( + await tx.execute(sql`UPDATE ${rel(t.schema, t.name)} AS t + SET ${sql.join(set.map((s) => sql`${sql.identifier(s.c.name)} = ${cast(s.v, s.c)}`), sql`, `)} + WHERE ${where} RETURNING ${rowJson(t)} AS r`) + ).r; + const changed = Object.fromEntries(set.map((s) => [s.c.name, before[s.c.name]])); + await audit(tx, actor, t, 'update', pk, changed, Object.fromEntries(set.map((s) => [s.c.name, after[s.c.name]]))); + return after; + }); +} + +export async function deleteRows(t: TableInfo, pks: Pk[], unlocked: boolean, actor: Actor) { + assertWritable(t, unlocked, 'delete'); + if (!pks.length || pks.length > 500) throw new DataError(400, 'Select 1–500 rows to delete.'); + const wheres = pks.map((pk) => pkWhere(t, pk)); + return write(async (tx) => { + let n = 0; + for (let i = 0; i < wheres.length; i++) { + const before = one<{ r: Record }>( + await tx.execute(sql`DELETE FROM ${rel(t.schema, t.name)} AS t WHERE ${wheres[i]} RETURNING ${rowJson(t)} AS r`) + )?.r; + if (!before) continue; + n++; + await audit(tx, actor, t, 'delete', pks[i], before, null); + } + return n; + }); +} + +/** Append CSV rows (header = column names) in one transaction: all or nothing. */ +export async function importRows( + t: TableInfo, + table: string[][], + unlocked: boolean, + actor: Actor +): Promise { + assertWritable(t, unlocked, 'insert'); + const [header, ...body] = table; + if (!header?.length) throw new DataError(400, 'The CSV is empty.'); + if (body.length > 20_000) throw new DataError(400, 'Import at most 20,000 rows at a time.'); + const names = header.map((h) => h.trim()); + const unknown = names.filter((n) => n && !t.columns.some((c) => c.name === n)); + if (unknown.length) throw new DataError(400, `Unknown column(s): ${unknown.join(', ')}.`); + if (new Set(names).size !== names.length) throw new DataError(400, 'Duplicate column in the header.'); + const prepared = body.map((cells, i) => { + if (cells.length !== names.length) + throw new DataError(400, `Row ${i + 2} has ${cells.length} cells, expected ${names.length}.`); + const values: Record = {}; + names.forEach((n, j) => { + if (!n) return; + const c = t.columns.find((x) => x.name === n)!; + if (c.readOnly) return; + // empty cell: column default when it has one, NULL otherwise (text stays '') + if (cells[j] === '' && (c.hasDefault || c.kind !== 'text')) return; + values[n] = cells[j]; + }); + try { + return assignments(t, values); + } catch (e) { + throw new DataError(400, `Row ${i + 2}: ${(e as Error).message}`); + } + }); + return write(async (tx) => { + for (const set of prepared) + await tx.execute( + set.length + ? sql`INSERT INTO ${rel(t.schema, t.name)} (${sql.join(set.map((s) => sql.identifier(s.c.name)), sql`, `)}) + VALUES (${sql.join(set.map((s) => cast(s.v, s.c)), sql`, `)})` + : sql`INSERT INTO ${rel(t.schema, t.name)} DEFAULT VALUES` + ); + await audit(tx, actor, t, 'import', null, null, { rows: prepared.length, columns: names }); + return prepared.length; + }); +} + +export const LIST_UI_TYPES = ['text', 'longtext', 'url', 'image', 'number', 'cents', 'date', 'datetime', 'bool', 'select', 'multiselect', 'json'] as const; + +/** Add a typed column to a runtime list (lists.* only) through meta.add_column(). */ +export async function addListColumn( + t: TableInfo, + input: { name: string; uiType: string; label?: string; options?: string[] }, + actor: Actor +) { + if (!t.isList || !t.listSlug || t.access !== 'open') + throw new DataError(403, 'Columns can only be added to lists.'); + if (!/^[a-z][a-z0-9_]{0,62}$/.test(input.name)) + throw new DataError(400, 'Column names use lowercase letters, digits and _ and start with a letter.'); + if (!(LIST_UI_TYPES as readonly string[]).includes(input.uiType)) + throw new DataError(400, 'Unknown column type.'); + if (t.columns.some((c) => c.name === input.name)) throw new DataError(409, 'That column already exists.'); + const col: Record = { name: input.name, ui_type: input.uiType }; + if (input.label?.trim()) col.label = input.label.trim().slice(0, 100); + if (input.uiType === 'select' || input.uiType === 'multiselect') { + const values = (input.options ?? []).map((o) => o.trim()).filter(Boolean).slice(0, 100); + if (input.uiType === 'select' && !values.length) throw new DataError(400, 'Give the select at least one option.'); + col.options = { values }; + } + await write(async (tx) => { + await tx.execute(sql`SELECT meta.add_column(${t.listSlug}, ${JSON.stringify(col)}::jsonb)`); + await audit(tx, actor, t, 'add_column', null, null, col); + }); +} diff --git a/src/lib/server/data-policy.test.ts b/src/lib/server/data-policy.test.ts new file mode 100644 index 0000000..3662264 --- /dev/null +++ b/src/lib/server/data-policy.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from 'vitest'; +import { + coerce, + hiddenColumn, + parseCsv, + parseFilter, + pgArrayLiteral, + plausibleIdent, + safeTypeName, + tableAccess, + toCsv, + typeKind, + writeAllowed, + type TableFacts +} from './data-policy'; + +const t = (schema: string, name: string, extra: Partial = {}): TableFacts => ({ + schema, + name, + relkind: 'r', + hasPrimaryKey: true, + canInsert: true, + canUpdate: true, + canDelete: true, + ...extra +}); + +describe('tableAccess', () => { + it('hides auth secrets, migrations and credential tables', () => { + for (const name of ['session', 'account', 'verification', 'jwks']) + expect(tableAccess(t('auth', name))).toBe('hidden'); + expect(tableAccess(t('drizzle', '__drizzle_migrations'))).toBe('hidden'); + expect(tableAccess(t('money', 'bank_sessions'))).toBe('hidden'); + expect(tableAccess(t('money', 'api_keys'))).toBe('hidden'); + expect(tableAccess(t('public', 'anything'))).toBe('hidden'); + }); + it('keeps auth.user, meta and legacy read-only', () => { + expect(tableAccess(t('auth', 'user'))).toBe('locked'); + expect(tableAccess(t('meta', 'tables'))).toBe('locked'); + expect(tableAccess(t('legacy', 'lists'))).toBe('locked'); + }); + it('locks views, tables without a key or without write grants', () => { + expect(tableAccess(t('energy', 'gas_storage_latest', { relkind: 'v' }))).toBe('locked'); + expect(tableAccess(t('money', 'x', { hasPrimaryKey: false }))).toBe('locked'); + expect( + tableAccess(t('ref', 'pokemon', { canInsert: false, canUpdate: false, canDelete: false })) + ).toBe('locked'); + }); + it('lists are open, app tables are core (unlock first)', () => { + expect(tableAccess(t('lists', 'books'))).toBe('open'); + expect(tableAccess(t('money', 'transactions'))).toBe('core'); + expect(tableAccess(t('books', 'sessions'))).toBe('core'); + expect(writeAllowed('open', false)).toBe(true); + expect(writeAllowed('core', false)).toBe(false); + expect(writeAllowed('core', true)).toBe(true); + expect(writeAllowed('locked', true)).toBe(false); + expect(writeAllowed('hidden', true)).toBe(false); + }); +}); + +describe('columns and identifiers', () => { + it('hides credential-like columns only', () => { + for (const c of ['password', 'access_token', 'refresh_token_expires_at', 'api_key', 'client_secret']) + expect(hiddenColumn(c)).toBe(true); + for (const c of ['key', 'session_id', 'iban', 'author', 'market_hash_name', 'r2_key']) + expect(hiddenColumn(c)).toBe(false); + }); + it('validates identifiers and type names', () => { + expect(plausibleIdent('done_at')).toBe(true); + expect(plausibleIdent('a"; drop table x; --')).toBe(false); + expect(plausibleIdent('')).toBe(false); + expect(safeTypeName('timestamp with time zone')).toBe(true); + expect(safeTypeName('numeric(12,2)')).toBe(true); + expect(safeTypeName('"books"."metadata_status"')).toBe(true); + expect(safeTypeName('text); drop table x; --')).toBe(false); + }); + it('maps types to editor kinds', () => { + expect(typeKind('integer', false)).toBe('number'); + expect(typeKind('numeric(12,2)', false)).toBe('number'); + expect(typeKind('text[]', false)).toBe('array'); + expect(typeKind('books.metadata_status', true)).toBe('enum'); + expect(typeKind('timestamp with time zone', false)).toBe('datetime'); + expect(typeKind('jsonb', false)).toBe('json'); + }); +}); + +describe('coerce', () => { + it('validates per kind', () => { + expect(coerce('number', '12.5', 'n')).toEqual({ ok: true, value: '12.5' }); + expect(coerce('number', 'abc', 'n').ok).toBe(false); + expect(coerce('number', '', 'n')).toEqual({ ok: true, value: null }); + expect(coerce('bool', 'yes', 'b')).toEqual({ ok: true, value: 'true' }); + expect(coerce('bool', 'maybe', 'b').ok).toBe(false); + expect(coerce('date', '2026-10-24', 'd').ok).toBe(true); + expect(coerce('date', '24/10', 'd').ok).toBe(false); + expect(coerce('json', '{"a":1}', 'j').ok).toBe(true); + expect(coerce('json', '{a:1}', 'j').ok).toBe(false); + expect(coerce('text', '', 't')).toEqual({ ok: true, value: '' }); + expect(coerce('text', null, 't')).toEqual({ ok: true, value: null }); + }); + it('builds array literals', () => { + expect(pgArrayLiteral(['a', 'b c', 'x"y'])).toBe('{"a","b c","x\\"y"}'); + expect(coerce('array', '["a","b"]', 'a')).toEqual({ ok: true, value: '{"a","b"}' }); + expect(coerce('array', 'a, b', 'a')).toEqual({ ok: true, value: '{"a","b"}' }); + }); +}); + +describe('filters and csv', () => { + it('parses filters', () => { + expect(parseFilter('name:contains:foo:bar')).toEqual({ column: 'name', op: 'contains', value: 'foo:bar' }); + expect(parseFilter('done_at:null')).toEqual({ column: 'done_at', op: 'null', value: '' }); + expect(parseFilter('x;y:eq:1')).toBeNull(); + expect(parseFilter('name:like:1')).toBeNull(); + }); + it('round-trips CSV', () => { + const csv = toCsv(['a', 'b'], [['x, y', 'he said "hi"'], [null, { k: 1 }], ['line\nbreak', 3]]); + expect(parseCsv(csv)).toEqual([ + ['a', 'b'], + ['x, y', 'he said "hi"'], + ['', '{"k":1}'], + ['line\nbreak', '3'] + ]); + expect(parseCsv('\ufeffa,b\r\n1,2')).toEqual([['a', 'b'], ['1', '2']]); + expect(() => parseCsv('a\n"open')).toThrow(); + }); +}); diff --git a/src/lib/server/data-policy.ts b/src/lib/server/data-policy.ts new file mode 100644 index 0000000..2d25820 --- /dev/null +++ b/src/lib/server/data-policy.ts @@ -0,0 +1,217 @@ +// Pure rules for the owner-only database browser at /lists/db: which tables and columns +// are visible or editable, identifier checks, value coercion and CSV. No DB access here, +// so it is unit tested directly (data-policy.test.ts). + +/** hidden: never listed or served. locked: always read-only. core: read-only until the + * owner unlocks editing for the table. open: freely editable (runtime lists). */ +export type Access = 'hidden' | 'locked' | 'core' | 'open'; + +export type TableFacts = { + schema: string; + name: string; + /** pg_class.relkind: r table, p partitioned, v view, m materialized view */ + relkind: string; + hasPrimaryKey: boolean; + canInsert: boolean; + canUpdate: boolean; + canDelete: boolean; +}; + +/** Schemas that are never shown: migration bookkeeping and Postgres internals. */ +const HIDDEN_SCHEMAS = new Set(['drizzle', 'information_schema', 'public']); +/** Better Auth: only the user table is visible (read-only); sessions, accounts (OAuth + * tokens, password hashes) and verification codes are hidden. */ +const AUTH_VISIBLE = new Set(['user']); +/** Individually hidden tables holding third-party credentials. */ +const HIDDEN_TABLES = new Set(['money.bank_sessions']); +/** Schemas whose tables are always read-only here (registry, rollback copies, auth). */ +const LOCKED_SCHEMAS = new Set(['meta', 'legacy', 'auth']); +const CREDENTIAL_NAME = + /(^|_)(passwords?|passwd|secrets?|tokens?|api_?keys?|private_?keys?|credentials?|salt|jwks|otp)($|_)/i; + +export function tableAccess(t: TableFacts): Access { + if (HIDDEN_SCHEMAS.has(t.schema) || t.schema.startsWith('pg_')) return 'hidden'; + if (t.schema === 'auth' && !AUTH_VISIBLE.has(t.name)) return 'hidden'; + if (HIDDEN_TABLES.has(`${t.schema}.${t.name}`) || CREDENTIAL_NAME.test(t.name)) return 'hidden'; + if (LOCKED_SCHEMAS.has(t.schema)) return 'locked'; + if (t.relkind !== 'r' && t.relkind !== 'p') return 'locked'; + if (!t.hasPrimaryKey) return 'locked'; + if (!t.canInsert && !t.canUpdate && !t.canDelete) return 'locked'; + return t.schema === 'lists' ? 'open' : 'core'; +} + +/** Columns that are never sent to the browser (tokens, password hashes, ...). */ +export function hiddenColumn(name: string): boolean { + return CREDENTIAL_NAME.test(name); +} + +/** Whether a write may go ahead: open tables always, core tables only when unlocked. */ +export function writeAllowed(access: Access, unlocked: boolean): boolean { + return access === 'open' || (access === 'core' && unlocked); +} + +/** Identifiers coming from the URL/body are looked up in the catalog; this is a cheap + * pre-filter so obviously bad input never reaches a query at all. */ +export function plausibleIdent(s: unknown): s is string { + return typeof s === 'string' && s.length > 0 && s.length <= 63 && /^[A-Za-z_][A-Za-z0-9_$]*$/.test(s); +} + +/** format_type() output that is safe to splice into a CAST (it comes from pg_catalog). */ +export function safeTypeName(type: string): boolean { + return /^[A-Za-z0-9_ ."(),[\]]{1,120}$/.test(type) && !type.includes('--') && !type.includes(';'); +} + +export type Kind = 'text' | 'number' | 'bool' | 'date' | 'datetime' | 'time' | 'json' | 'enum' | 'array' | 'uuid' | 'other'; + +export function typeKind(type: string, isEnum: boolean): Kind { + if (type.endsWith('[]')) return 'array'; + if (isEnum) return 'enum'; + if (/^(smallint|integer|bigint|real|double precision|numeric|money)/.test(type)) return 'number'; + if (type === 'boolean') return 'bool'; + if (type === 'date') return 'date'; + if (type.startsWith('timestamp')) return 'datetime'; + if (type.startsWith('time')) return 'time'; + if (type === 'json' || type === 'jsonb') return 'json'; + if (type === 'uuid') return 'uuid'; + if (/^(text|character|"char"|citext|name)/.test(type)) return 'text'; + return 'other'; +} + +/** Postgres array literal for a list of strings, e.g. ['a', 'b c'] -> {"a","b c"}. */ +export function pgArrayLiteral(items: unknown[]): string { + return `{${items + .map((v) => + v === null || v === undefined + ? 'NULL' + : `"${String(typeof v === 'object' ? JSON.stringify(v) : v).replace(/["\\]/g, '\\$&')}"` + ) + .join(',')}}`; +} + +export type Coerced = { ok: true; value: string | null } | { ok: false; error: string }; + +/** + * Turn a value from the editor/CSV into the text that is bound as a parameter and + * CAST to the column type by Postgres (which does the final validation). null = NULL. + * Empty input means NULL for every non-text column. + */ +export function coerce(kind: Kind, raw: unknown, label: string): Coerced { + if (raw === null || raw === undefined) return { ok: true, value: null }; + if (kind === 'array') { + let items: unknown = raw; + if (typeof raw === 'string') { + const s = raw.trim(); + if (!s) return { ok: true, value: null }; + if (s.startsWith('{')) return { ok: true, value: s }; // already a Postgres literal + try { + items = JSON.parse(s); + } catch { + items = s.split(',').map((x) => x.trim()).filter(Boolean); + } + } + if (!Array.isArray(items)) return { ok: false, error: `${label} must be a list.` }; + return { ok: true, value: pgArrayLiteral(items) }; + } + if (kind === 'json') { + if (typeof raw !== 'string') return { ok: true, value: JSON.stringify(raw) }; + if (!raw.trim()) return { ok: true, value: null }; + try { + JSON.parse(raw); + } catch { + return { ok: false, error: `${label} is not valid JSON.` }; + } + return { ok: true, value: raw }; + } + const s = typeof raw === 'string' ? raw : String(raw); + if (kind === 'text' || kind === 'other') return { ok: true, value: s }; + const t = s.trim(); + if (!t) return { ok: true, value: null }; + switch (kind) { + case 'number': + if (!/^[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?$/i.test(t.replace(/_/g, ''))) + return { ok: false, error: `${label} must be a number.` }; + return { ok: true, value: t.replace(/_/g, '') }; + case 'bool': { + const v = t.toLowerCase(); + if (['true', 't', 'yes', 'y', '1', 'on'].includes(v)) return { ok: true, value: 'true' }; + if (['false', 'f', 'no', 'n', '0', 'off'].includes(v)) return { ok: true, value: 'false' }; + return { ok: false, error: `${label} must be true or false.` }; + } + case 'date': + if (!/^\d{4}-\d{2}-\d{2}/.test(t)) return { ok: false, error: `${label} must be a date (YYYY-MM-DD).` }; + return { ok: true, value: t }; + case 'datetime': + if (Number.isNaN(Date.parse(t)) && !/^\d{4}-\d{2}-\d{2}/.test(t)) + return { ok: false, error: `${label} must be a date and time.` }; + return { ok: true, value: t }; + case 'uuid': + if (!/^[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}$/i.test(t)) + return { ok: false, error: `${label} must be a UUID.` }; + return { ok: true, value: t }; + default: + return { ok: true, value: t }; + } +} + +/** Filter operators accepted by the grid (?f=::). */ +export const FILTER_OPS = ['contains', 'eq', 'ne', 'gt', 'gte', 'lt', 'lte', 'null', 'notnull'] as const; +export type FilterOp = (typeof FILTER_OPS)[number]; +export type Filter = { column: string; op: FilterOp; value: string }; + +export function parseFilter(s: string): Filter | null { + const i = s.indexOf(':'); + const j = i < 0 ? -1 : s.indexOf(':', i + 1); + if (i < 1) return null; + const column = s.slice(0, i); + const op = (j < 0 ? s.slice(i + 1) : s.slice(i + 1, j)) as FilterOp; + const value = j < 0 ? '' : s.slice(j + 1); + if (!plausibleIdent(column) || !FILTER_OPS.includes(op) || value.length > 500) return null; + return { column, op, value }; +} + +// ---- CSV (RFC 4180) ---- + +export function csvCell(v: unknown): string { + if (v === null || v === undefined) return ''; + const s = typeof v === 'object' ? JSON.stringify(v) : String(v); + return /[",\r\n]/.test(s) || /^\s|\s$/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; +} + +export function toCsv(header: string[], rows: unknown[][]): string { + return [header, ...rows].map((r) => r.map(csvCell).join(',')).join('\r\n') + '\r\n'; +} + +export function parseCsv(text: string): string[][] { + const rows: string[][] = []; + let row: string[] = []; + let cell = ''; + let quoted = false; + let i = text.charCodeAt(0) === 0xfeff ? 1 : 0; // BOM + for (; i < text.length; i++) { + const ch = text[i]; + if (quoted) { + if (ch === '"') { + if (text[i + 1] === '"') { + cell += '"'; + i++; + } else quoted = false; + } else cell += ch; + } else if (ch === '"' && cell === '') quoted = true; + else if (ch === ',') { + row.push(cell); + cell = ''; + } else if (ch === '\n' || ch === '\r') { + if (ch === '\r' && text[i + 1] === '\n') i++; + row.push(cell); + rows.push(row); + row = []; + cell = ''; + } else cell += ch; + } + if (quoted) throw new Error('CSV has an unterminated quoted field.'); + if (cell !== '' || row.length) { + row.push(cell); + rows.push(row); + } + return rows.filter((r) => !(r.length === 1 && r[0] === '')); +} diff --git a/src/lib/server/data-route.ts b/src/lib/server/data-route.ts new file mode 100644 index 0000000..9f7ae75 --- /dev/null +++ b/src/lib/server/data-route.ts @@ -0,0 +1,47 @@ +// Request helpers shared by the /lists/db routes. Every route calls ownerTable() (or +// requireOwner) itself, so access never depends on the layout load having run. +import { error, json } from '@sveltejs/kit'; +import { requireOwner } from '#lib/server/owner.js'; +import { parseFilter, type Filter } from '#lib/server/data-policy.js'; +import { DataError, getTable, type Actor, type RowQuery } from '#lib/server/data-editor.js'; + +type Event = { locals: App.Locals; params: Record; request: Request; url: URL }; + +export async function ownerTable(event: Event) { + const user = requireOwner(event); + const table = await getTable(event.params.schema ?? '', event.params.table ?? ''); + if (!table) error(404, 'Not Found'); + const actor: Actor = { id: user.id, email: user.email }; + return { table, actor }; +} + +/** Core (app) tables are read-only until the owner unlocks editing in the UI, which sends + * this header with every write; the server refuses core-table writes without it. */ +export const unlocked = (request: Request) => request.headers.get('x-data-unlock') === '1'; + +export function rowQuery(url: URL): RowQuery { + const p = url.searchParams; + const filters = p + .getAll('f') + .slice(0, 20) + .map(parseFilter) + .filter((f): f is Filter => f !== null); + const limit = Number(p.get('limit') ?? 50); + const page = Number(p.get('page') ?? 1); + const safeLimit = [25, 50, 100, 250, 500].includes(limit) ? limit : 50; + return { + q: (p.get('q') ?? '').slice(0, 200), + filters, + sort: p.get('sort') || undefined, + dir: p.get('dir') === 'desc' ? 'desc' : 'asc', + limit: safeLimit, + offset: Number.isSafeInteger(page) && page > 1 ? (page - 1) * safeLimit : 0 + }; +} + +export function failure(e: unknown) { + if (e instanceof DataError) return json({ message: e.message }, { status: e.status }); + throw e; +} + +export const noStore = { 'cache-control': 'private, no-store', 'x-robots-tag': 'noindex, nofollow' }; diff --git a/src/lib/server/lists.ts b/src/lib/server/lists.ts index 2edf996..e94aab5 100644 --- a/src/lib/server/lists.ts +++ b/src/lib/server/lists.ts @@ -44,6 +44,7 @@ export type ListSummary = { checked: number; }; export type ListView = Omit & { + tableName: string; columns: ListColumn[]; items: ListItem[]; }; @@ -144,6 +145,7 @@ export async function getList(slug: string): Promise { slug: t.slug, pictureUrl: t.pictureUrl, isProgress: t.hasDone, + tableName: t.tableName, columns, items: [...rows].map((r) => toItem(r, columns)) }; diff --git a/src/lib/server/owner.ts b/src/lib/server/owner.ts index e6fd9ba..6ad30fc 100644 --- a/src/lib/server/owner.ts +++ b/src/lib/server/owner.ts @@ -22,7 +22,8 @@ export function requireOwner(event: OwnerEvent) { export const ownerOnlyPrefixes = [ '/trackers/health', '/trackers/tasks', - '/trackers/books/files' + '/trackers/books/files', + '/lists/db' ] as const; /** diff --git a/src/lib/server/slugify.ts b/src/lib/server/slugify.ts index e59465b..6dbd6d1 100644 --- a/src/lib/server/slugify.ts +++ b/src/lib/server/slugify.ts @@ -28,10 +28,13 @@ export async function uniqueSlug(title: string): Promise { return slug; } +/** /lists/db is the database browser, so no list may use that slug. */ +export const RESERVED_LIST_SLUGS = new Set(['db']); + export async function uniqueListSlug(name: string): Promise { const base = slugify(name, 'list'); - let slug = base; - let suffix = 2; + let slug = RESERVED_LIST_SLUGS.has(base) ? `${base}-2` : base; + let suffix = RESERVED_LIST_SLUGS.has(base) ? 3 : 2; while ( (await db.select({ id: metaTables.id }).from(metaTables).where(eq(metaTables.slug, slug))) .length > 0 diff --git a/src/routes/lists/+page.server.ts b/src/routes/lists/+page.server.ts index 7b36dec..edbe78d 100644 --- a/src/routes/lists/+page.server.ts +++ b/src/routes/lists/+page.server.ts @@ -2,10 +2,21 @@ import { requireAdmin } from '#lib/server/authorization.js'; import { uniqueListSlug } from '#lib/server/slugify.js'; import { listImageUrl } from '#lib/server/r2.js'; import { createList, listSummaries } from '#lib/server/lists.js'; +import { listTables, type TableSummary } from '#lib/server/data-editor.js'; +import { isOwner } from '#lib/server/owner.js'; import { fail, redirect } from '@sveltejs/kit'; import type { Actions, PageServerLoad } from './$types'; -export const load: PageServerLoad = async () => ({ lists: await listSummaries() }); +export const load: PageServerLoad = async (event) => { + const owner = isOwner(event); + // The database browser (every schema/table) is only ever loaded for the owner. + const [lists, tables] = await Promise.all([ + listSummaries(), + owner ? listTables() : Promise.resolve([] as TableSummary[]) + ]); + if (owner) event.setHeaders({ 'cache-control': 'private, no-store' }); + return { lists, tables }; +}; export const actions: Actions = { newList: async (event) => { diff --git a/src/routes/lists/+page.svelte b/src/routes/lists/+page.svelte index 3d386c7..2fb79fd 100644 --- a/src/routes/lists/+page.svelte +++ b/src/routes/lists/+page.svelte @@ -14,6 +14,24 @@ import { Input } from '#lib/components/ui/input/index.js'; import type { PageProps } from './$types'; let { data, form }: PageProps = $props(); + let tableFilter = $state(''); + const schemas = $derived.by(() => { + const q = tableFilter.trim().toLowerCase(); + const groups = new Map(); + for (const t of data.tables) { + if (q && !`${t.schema}.${t.name} ${t.displayName}`.toLowerCase().includes(q)) continue; + groups.set(t.schema, [...(groups.get(t.schema) ?? []), t]); + } + // lists first, then alphabetical + return [...groups].sort(([a], [b]) => (a === 'lists' ? -1 : b === 'lists' ? 1 : a.localeCompare(b))); + }); + const totalRows = $derived(data.tables.reduce((n, t) => n + (t.rows ?? 0), 0)); + const badge = (access: string, relkind: string) => + access === 'open' + ? { text: 'editable', cls: 'border-emerald-600/40 text-emerald-500' } + : access === 'core' + ? { text: 'app table', cls: 'border-amber-500/40 text-amber-500' } + : { text: relkind === 'v' || relkind === 'm' ? 'view' : 'read-only', cls: 'border-[var(--site-border)] text-[var(--site-muted)]' }; let uploadError = $state(''); let uploading = $state(false); const uploadPicture: SubmitFunction = async ({ formData, cancel }) => { @@ -137,4 +155,51 @@ {/each} + + {#if data.isOwner && data.tables.length} +
+
+
+

database

+

+ {data.tables.length} tables in {new Set(data.tables.map((t) => t.schema)).size} schemas, ~{totalRows.toLocaleString('en-GB')} rows. Only you can see this. Lists are editable; app tables are read-only until you unlock them; a new list is a new table. +

+
+ +
+ {#each schemas as [schema, tables] (schema)} +
+ + {schema} + {tables.length} {tables.length === 1 ? 'table' : 'tables'} + + +
+ {:else} +

no tables match

+ {/each} +
+ {/if} diff --git a/src/routes/lists/[slug]/+page.server.ts b/src/routes/lists/[slug]/+page.server.ts index 9662f47..ccbe012 100644 --- a/src/routes/lists/[slug]/+page.server.ts +++ b/src/routes/lists/[slug]/+page.server.ts @@ -3,7 +3,7 @@ import { ORIGIN } from '$app/env/private'; import { metaTables } from '#lib/server/db/schema/index.js'; import { requireAdmin } from '#lib/server/authorization.js'; import { deleteObject, listImageKey, listImageUrl } from '#lib/server/r2.js'; -import { validSlug } from '#lib/server/slugify.js'; +import { RESERVED_LIST_SLUGS, validSlug } from '#lib/server/slugify.js'; import { archiveList, deleteItem, @@ -109,7 +109,7 @@ export const actions: Actions = { const picture = form.get('picture'); if (!name || name.length > 200) return fail(400, { message: 'Enter a name of 1–200 characters' }); - if (!validSlug(slug) || slug.length > 63) + if (!validSlug(slug) || slug.length > 63 || RESERVED_LIST_SLUGS.has(slug)) return fail(400, { message: 'Use a lowercase slug with letters, numbers, and hyphens.' }); if (picture instanceof File && picture.size > 0) return fail(400, { message: 'Enable JavaScript to upload an image.' }); diff --git a/src/routes/lists/[slug]/+page.svelte b/src/routes/lists/[slug]/+page.svelte index e795a19..f5e7594 100644 --- a/src/routes/lists/[slug]/+page.svelte +++ b/src/routes/lists/[slug]/+page.svelte @@ -99,6 +99,10 @@

+ {#if data.isOwner}table view{/if} {#if data.isAdmin} + {:else} + This table belongs to the {t.schema} app, so it is read-only by default. + + {/if} + + {/if} + +
+ onSearch(e.currentTarget.value)} + aria-label="Search rows" + /> +
+ + + {#if fOp !== 'null' && fOp !== 'notnull'} + + {/if} + +
+
+ {#if editable && t.canInsert} + + + + {/if} + {#if t.isList && t.access === 'open'} + + {/if} + export CSV + {#if editable && t.canDelete && selectedRows.length} + + {/if} +
+
+ + {#if filters.length} +
+ {#each filters as f, i (f + i)} + + {/each} +
+ {/if} + {#if data.queryError}{/if} + {#if message} +

{message.text}

+ {/if} + +
+ + + + {#if t.primaryKey.length} + + {/if} + {#each cols as c (c.name)} + {@const sorted = params.get('sort') === c.name} + + {/each} + {#if editable && t.canDelete}{/if} + + + + {#each rows as row, r (keyOf(row) + r)} + + {#if t.primaryKey.length} + + {/if} + {#each cols as c, ci (c.name)} + {@const v = row[c.name]} + {#if editing && editing.r === r && editing.c === ci} + + {:else} + + {/if} + {/each} + {#if editable && t.canDelete} + + {/if} + + {:else} + + {/each} + +
+ 0 && selectedRows.length === rows.length} + onchange={(e) => (selected = e.currentTarget.checked ? Object.fromEntries(rows.map((r) => [keyOf(r), true])) : {})} + /> + + +
+ {shortType(c.type)}{c.nullable ? '' : ' · req'}{c.fk ? ` → ${c.fk.table}` : ''}{c.readOnly ? ' · auto' : ''} +
+
+ + + +
Enter{c.kind === 'json' || c.uiType === 'longtext' ? ' + Ctrl' : ''} save · Esc cancel · Tab next
+ 		(active = { r, c: ci })} + ondblclick={() => startEdit(r, ci)} + onkeydown={(e) => cellKey(e, r, ci)} + > + {#if v === null}null + {:else if c.fk && data.fkLabels[c.name]?.[String(v)]}{data.fkLabels[c.name][String(v)]} + {:else if (c.uiType === 'url' || c.uiType === 'image') && typeof v === 'string' && /^https?:\/\//.test(v)} + {display(c, v)} + {:else}{display(c, v)}{/if} + + +
no rows
+
+ +
+ + page {pageNo}{pages ? ` of ${pages.toLocaleString('en-GB')}` : ''} + + + ↑↓←→ move · Enter edit · Space select · / search · n new row +
+ + + + + + Unlock editing {t.schema}.{t.name}? + + This is an app table, not a list. Edits change what the {t.schema} pages show and could break them. Every change is written to the edit log. Editing stays unlocked for this table until you lock it again or close the tab. + + + + cancel + { setUnlocked(true); unlockOpen = false; }}>unlock editing + + + + + + + + Delete {deleteTargets.length} row{deleteTargets.length === 1 ? '' : 's'}? + + From {t.schema}.{t.name}. This can't be undone here (the old values are kept in the edit log). + + + + cancel + delete + + + + + + + + Import {importFile?.name}? + + Appends every row to {t.schema}.{t.name}. The first line must hold column names; empty cells use the column default. If any row fails, nothing is imported. + + + + { importFile = null; if (fileInput) fileInput.value = ''; }}>cancel + import + + + + + + + new row in {t.schema}.{t.name} + {#if newError}{/if} +
+ {#each insertable as c, i (c.name)} + + {/each} + +
+ + + + + + new column + {#if colError}{/if} +
+ + + + {#if colType === 'select' || colType === 'multiselect'} + + {/if} + +
+ + diff --git a/src/routes/lists/db/[schema]/[table]/columns/+server.ts b/src/routes/lists/db/[schema]/[table]/columns/+server.ts new file mode 100644 index 0000000..c63f43c --- /dev/null +++ b/src/routes/lists/db/[schema]/[table]/columns/+server.ts @@ -0,0 +1,25 @@ +import { json } from '@sveltejs/kit'; +import { addListColumn } from '#lib/server/data-editor.js'; +import { failure, noStore, ownerTable } from '#lib/server/data-route.js'; +import type { RequestHandler } from './$types'; + +/** Add a column to a runtime list (schema `lists` only, via meta.add_column()). */ +export const POST: RequestHandler = async (event) => { + const { table, actor } = await ownerTable(event); + const b = (await event.request.json().catch(() => ({}))) as Record; + try { + await addListColumn( + table, + { + name: String(b.name ?? '').trim(), + uiType: String(b.uiType ?? ''), + label: typeof b.label === 'string' ? b.label : undefined, + options: Array.isArray(b.options) ? b.options.map(String) : undefined + }, + actor + ); + return json({ ok: true }, { status: 201, headers: noStore }); + } catch (e) { + return failure(e); + } +}; diff --git a/src/routes/lists/db/[schema]/[table]/export/+server.ts b/src/routes/lists/db/[schema]/[table]/export/+server.ts new file mode 100644 index 0000000..3fb7dc0 --- /dev/null +++ b/src/routes/lists/db/[schema]/[table]/export/+server.ts @@ -0,0 +1,23 @@ +import { exportRows } from '#lib/server/data-editor.js'; +import { toCsv } from '#lib/server/data-policy.js'; +import { failure, noStore, ownerTable, rowQuery } from '#lib/server/data-route.js'; +import type { RequestHandler } from './$types'; + +/** CSV of every row matching the current search/filters/sort (header = column names). */ +export const GET: RequestHandler = async (event) => { + const { table } = await ownerTable(event); + try { + const rows = await exportRows(table, { ...rowQuery(event.url), limit: undefined, offset: 0 }); + const csv = toCsv(table.columns.map((c) => c.name), rows); + const stamp = new Date().toISOString().slice(0, 10); + return new Response(csv, { + headers: { + ...noStore, + 'content-type': 'text/csv; charset=utf-8', + 'content-disposition': `attachment; filename="${table.schema}.${table.name}-${stamp}.csv"` + } + }); + } catch (e) { + return failure(e); + } +}; diff --git a/src/routes/lists/db/[schema]/[table]/import/+server.ts b/src/routes/lists/db/[schema]/[table]/import/+server.ts new file mode 100644 index 0000000..f66123d --- /dev/null +++ b/src/routes/lists/db/[schema]/[table]/import/+server.ts @@ -0,0 +1,26 @@ +import { json } from '@sveltejs/kit'; +import { DataError, importRows } from '#lib/server/data-editor.js'; +import { parseCsv } from '#lib/server/data-policy.js'; +import { failure, noStore, ownerTable, unlocked } from '#lib/server/data-route.js'; +import type { RequestHandler } from './$types'; + +/** Append rows from a CSV body (text/csv, at most 10 MB). All rows or none. */ +export const POST: RequestHandler = async (event) => { + const { table, actor } = await ownerTable(event); + try { + const length = Number(event.request.headers.get('content-length') ?? 0); + if (length > 10 * 1024 * 1024) throw new DataError(413, 'CSV files are limited to 10 MB.'); + const text = await event.request.text(); + if (text.length > 10 * 1024 * 1024) throw new DataError(413, 'CSV files are limited to 10 MB.'); + let rows: string[][]; + try { + rows = parseCsv(text); + } catch (e) { + throw new DataError(400, (e as Error).message); + } + const inserted = await importRows(table, rows, unlocked(event.request), actor); + return json({ inserted }, { headers: noStore }); + } catch (e) { + return failure(e); + } +}; diff --git a/src/routes/lists/db/[schema]/[table]/lookup/+server.ts b/src/routes/lists/db/[schema]/[table]/lookup/+server.ts new file mode 100644 index 0000000..948dc6b --- /dev/null +++ b/src/routes/lists/db/[schema]/[table]/lookup/+server.ts @@ -0,0 +1,13 @@ +import { json } from '@sveltejs/kit'; +import { fkOptions } from '#lib/server/data-editor.js'; +import { noStore, ownerTable } from '#lib/server/data-route.js'; +import type { RequestHandler } from './$types'; + +/** Options for a foreign-key column: rows of the referenced table as {value, label}. */ +export const GET: RequestHandler = async (event) => { + const { table } = await ownerTable(event); + const column = table.columns.find((c) => c.name === event.url.searchParams.get('column')); + if (!column?.fk) return json({ options: [] }, { headers: noStore }); + const q = (event.url.searchParams.get('q') ?? '').slice(0, 100); + return json({ options: await fkOptions(column, { q }) }, { headers: noStore }); +}; diff --git a/src/routes/lists/db/[schema]/[table]/rows/+server.ts b/src/routes/lists/db/[schema]/[table]/rows/+server.ts new file mode 100644 index 0000000..f14f742 --- /dev/null +++ b/src/routes/lists/db/[schema]/[table]/rows/+server.ts @@ -0,0 +1,56 @@ +import { json } from '@sveltejs/kit'; +import { deleteRows, insertRow, readRows, updateRow } from '#lib/server/data-editor.js'; +import { failure, noStore, ownerTable, rowQuery, unlocked } from '#lib/server/data-route.js'; +import type { RequestHandler } from './$types'; + +const isObject = (v: unknown): v is Record => + typeof v === 'object' && v !== null && !Array.isArray(v); + +async function body(request: Request): Promise> { + const b = await request.json().catch(() => null); + return isObject(b) ? b : {}; +} + +export const GET: RequestHandler = async (event) => { + const { table } = await ownerTable(event); + try { + return json(await readRows(table, rowQuery(event.url)), { headers: noStore }); + } catch (e) { + return failure(e); + } +}; + +export const POST: RequestHandler = async (event) => { + const { table, actor } = await ownerTable(event); + const b = await body(event.request); + try { + const row = await insertRow(table, isObject(b.values) ? b.values : {}, unlocked(event.request), actor); + return json({ row }, { status: 201, headers: noStore }); + } catch (e) { + return failure(e); + } +}; + +export const PATCH: RequestHandler = async (event) => { + const { table, actor } = await ownerTable(event); + const b = await body(event.request); + try { + if (!isObject(b.pk) || !isObject(b.values)) return json({ message: 'Bad request.' }, { status: 400 }); + const row = await updateRow(table, b.pk, b.values, unlocked(event.request), actor); + return json({ row }, { headers: noStore }); + } catch (e) { + return failure(e); + } +}; + +export const DELETE: RequestHandler = async (event) => { + const { table, actor } = await ownerTable(event); + const b = await body(event.request); + try { + const pks = Array.isArray(b.pks) ? b.pks.filter(isObject) : []; + const deleted = await deleteRows(table, pks, unlocked(event.request), actor); + return json({ deleted }, { headers: noStore }); + } catch (e) { + return failure(e); + } +};