worldwide todo list: public todo list of 68 real, sourced problems (planet to street) at /projects/worldwide-todo. worldtodo schema (categories, tasks, change_requests), idempotent seed on deploy, public add/edit/delete requests with Turnstile + salted daily-rotating IP-hash rate limit, owner-only review queue (approve, edit & approve, reject) with live updates over /ws/worldtodo(-owner), JSON API, detail pages with OSM map and history; listed in projects, header, /start and sitemap
Some checks failed
Deploy production / deploy (push) Failing after 9s

This commit is contained in:
2026-10-26 01:25:59 +00:00
parent 0ee981a96b
commit 5e812271e1
58 changed files with 47689 additions and 9 deletions

View File

@@ -0,0 +1,33 @@
import type { TaskFields } from '#lib/worldtodo/shared.js';
import { publishQueue, publishTask } from './live';
import * as store from './store';
import { plain } from './validate';
// Owner decisions (queue actions, API, direct edits). Applied in one transaction in the
// store, then published on the realtime channels.
export const ownerNote = (v: unknown) =>
plain(typeof v === 'string' ? v : '', true).slice(0, 1000) || null;
async function publishApplied(applied: store.Applied) {
const found = applied.type === 'task.deleted' ? null : await store.getTask(applied.slug);
publishTask(applied.type, applied.slug, found?.task ?? null);
publishQueue(await store.pendingTotal());
}
export async function approveRequest(id: number, note: unknown, tweaked: TaskFields | null = null) {
const applied = await store.approve(id, ownerNote(note), tweaked);
await publishApplied(applied);
return applied;
}
export async function rejectRequest(id: number, note: unknown) {
await store.reject(id, ownerNote(note));
publishQueue(await store.pendingTotal());
}
export async function editTaskDirectly(slug: string, fields: TaskFields, note: unknown) {
const applied = await store.ownerEdit(slug, fields, ownerNote(note));
await publishApplied(applied);
return applied;
}

View File

@@ -0,0 +1,37 @@
import { describe, expect, it } from 'vitest';
import { PER_DAY, PER_HOUR, createLimiter } from './limit';
describe('worldtodo rate limiter', () => {
it('allows 5 per hour, then answers with a Retry-After', () => {
let t = Date.UTC(2026, 9, 26, 10, 0, 0);
const l = createLimiter(() => t);
for (let i = 0; i < PER_HOUR; i++) expect(l.hit('1.2.3.4')).toBeNull();
const wait = l.hit('1.2.3.4');
expect(wait).toBeGreaterThan(3500);
expect(wait).toBeLessThanOrEqual(3600);
expect(l.hit('5.6.7.8')).toBeNull(); // other clients unaffected
t += 3_600_001;
expect(l.hit('1.2.3.4')).toBeNull();
});
it('caps at 20 per day', () => {
let t = Date.UTC(2026, 9, 26, 0, 30, 0);
const l = createLimiter(() => t);
for (let i = 0; i < PER_DAY; i++) {
expect(l.hit('9.9.9.9')).toBeNull();
t += 13 * 60_000; // ~4.6 per hour, under the hourly cap
}
expect(l.hit('9.9.9.9')).not.toBeNull();
});
it('never keeps the raw IP and rotates the salt every UTC day', () => {
let t = Date.UTC(2026, 9, 26, 23, 59, 0);
const l = createLimiter(() => t);
const k1 = l.keyOf('1.2.3.4');
expect(k1).not.toContain('1.2.3.4');
expect(l.keyOf('1.2.3.4')).toBe(k1);
t += 120_000; // next UTC day
expect(l.keyOf('1.2.3.4')).not.toBe(k1);
expect(l.size()).toBe(0);
});
});

View File

@@ -0,0 +1,66 @@
import { createHmac, randomBytes } from 'node:crypto';
// Rate limiting for public change requests: 5 per hour and 20 per day per client.
// The client key is an HMAC of the IP address with a random salt that lives only in this
// process's memory and rotates every UTC day, so a key can't be linked to an IP (or to
// yesterday's key) and nothing is ever written to Postgres or logs. Counters are per process
// and reset on restart, like the site's other limiters.
export const PER_HOUR = 5;
export const PER_DAY = 20;
const HOUR = 3_600_000;
const DAY = 86_400_000;
const MAX_KEYS = 50_000;
type State = { day: string; salt: Buffer; hits: Map<string, number[]> };
export function createLimiter(now: () => number = Date.now) {
let state: State | null = null;
const current = () => {
const day = new Date(now()).toISOString().slice(0, 10);
if (!state || state.day !== day) state = { day, salt: randomBytes(32), hits: new Map() };
return state;
};
const keyOf = (ip: string) =>
createHmac('sha256', current().salt).update(ip).digest('base64url').slice(0, 22);
return {
/** The salted hash for an IP (exposed for tests; never stored). */
keyOf,
/**
* Count one attempt. Returns null when allowed, or the seconds to wait (Retry-After).
* Blocked attempts are not counted.
*/
hit(ip: string): number | null {
const s = current();
const key = keyOf(ip);
const t = now();
const list = (s.hits.get(key) ?? []).filter((x) => t - x < DAY);
const lastHour = list.filter((x) => t - x < HOUR);
if (lastHour.length >= PER_HOUR)
return Math.max(1, Math.ceil((lastHour[0] + HOUR - t) / 1000));
if (list.length >= PER_DAY) return Math.max(1, Math.ceil((list[0] + DAY - t) / 1000));
list.push(t);
if (!s.hits.has(key) && s.hits.size >= MAX_KEYS) s.hits.delete(s.hits.keys().next().value!);
s.hits.set(key, list);
return null;
},
/** Number of tracked keys (tests). */
size: () => current().hits.size
};
}
export const limiter = createLimiter();
/** The client IP behind Cloudflare/Caddy, same order as server.js and the listening stream. */
export function clientIp(request: Request, getClientAddress: () => string): string {
const h = request.headers;
const forwarded =
h.get('cf-connecting-ip') ?? h.get('x-forwarded-for')?.split(',')[0]?.trim() ?? '';
if (forwarded) return forwarded;
try {
return getClientAddress();
} catch {
return 'unknown';
}
}

View File

@@ -0,0 +1,86 @@
import { registerChannel, type Socket } from '#lib/server/ws/hub.js';
import { auth } from '#lib/server/auth.js';
import { isOwner } from '#lib/server/owner.js';
import type { LiveEvent, PublicTask } from '#lib/worldtodo/shared.js';
// Realtime for the Worldwide Todo List on the site WebSocket hub:
// - `worldtodo` (public, read-only): every approved change, as
// { type: 'task.added' | 'task.updated' | 'task.deleted', slug, task | null, at }.
// - `worldtodo-owner` (owner session only): { type: 'queue.changed', pending } when a request
// is filed or decided, so the queue page and the header badge refresh without polling.
// Clients send nothing; any message closes the socket. Pages fall back to polling (60 s)
// when the socket can't connect (e.g. `vite dev`, which has no Bun.serve).
const MAX_PER_IP = 8;
const publicSockets = new Set<Socket>();
const ownerSockets = new Set<Socket>();
const perIp = new Map<string, number>();
function track(set: Set<Socket>) {
return {
open(ws: Socket) {
set.add(ws);
},
message(ws: Socket) {
ws.close(1008, 'read only');
},
close(ws: Socket) {
set.delete(ws);
const n = (perIp.get(ws.data.ip) ?? 1) - 1;
if (n <= 0) perIp.delete(ws.data.ip);
else perIp.set(ws.data.ip, n);
}
};
}
function admit(ip: string): Response | null {
const n = perIp.get(ip) ?? 0;
if (n >= MAX_PER_IP) return new Response('Too Many Connections', { status: 429 });
perIp.set(ip, n + 1);
return null;
}
let registered = false;
export function registerWorldtodoChannels() {
if (registered) return;
registered = true;
registerChannel('worldtodo', {
async authorize(_request, _url, ip) {
return admit(ip) ?? {};
},
...track(publicSockets)
});
registerChannel('worldtodo-owner', {
async authorize(request, _url, ip) {
const session = await auth.api.getSession({ headers: request.headers });
if (!isOwner({ locals: { user: session?.user ?? null } as App.Locals }))
return new Response('Not Found', { status: 404 });
return admit(ip) ?? {};
},
...track(ownerSockets)
});
}
function send(set: Set<Socket>, msg: unknown) {
const text = JSON.stringify(msg);
for (const ws of set) {
try {
ws.send(text);
} catch {
/* closing socket */
}
}
}
export function publishTask(type: LiveEvent['type'], slug: string, task: PublicTask | null) {
send(publicSockets, {
type,
slug,
task: type === 'task.deleted' ? null : task,
at: new Date().toISOString()
} satisfies LiveEvent);
}
export function publishQueue(pending: number) {
send(ownerSockets, { type: 'queue.changed', pending });
}

View File

@@ -0,0 +1,116 @@
import { changedFields, type TaskFields } from '#lib/worldtodo/shared.js';
import { limiter } from './limit';
import { publishQueue } from './live';
import * as store from './store';
import { turnstileConfigured, verifyTurnstile } from './turnstile';
import { looksLikeBot, parseRequest, type FieldErrors } from './validate';
// Filing a public change request (shared by the form action and the JSON API). Order:
// bot checks → validation (no network, so a typo never burns a Turnstile token or a rate
// slot) → rate limit (before Siteverify, so floods never reach Cloudflare) → global queue
// cap → Turnstile → insert. Nothing about the submitter is stored except the optional name.
export const MAX_PENDING = 200;
export type SubmitResult =
| { ok: true; id: number }
| { ok: false; status: number; message: string; errors?: FieldErrors; retryAfter?: number };
export async function submitRequest(opts: {
input: Record<string, unknown>;
honeypot: unknown;
startedAt: unknown;
token: string;
ip: string;
now?: number;
}): Promise<SubmitResult> {
if (!turnstileConfigured())
return { ok: false, status: 503, message: 'Requests are switched off right now.' };
if (looksLikeBot(opts.honeypot, opts.startedAt, opts.now))
return {
ok: false,
status: 400,
message: 'That was a bit quick. Wait a few seconds and send it again.'
};
// API edits may be partial, so validate them on top of the task as it is now.
let base: Record<string, unknown> | undefined;
if (opts.input.kind === 'edit' && typeof opts.input.task === 'string' && opts.input.task.trim()) {
const current = await store.getTask(opts.input.task.trim());
if (current) base = { ...store.currentFields(current.task) };
}
const parsed = parseRequest(opts.input, base);
if (!parsed.ok)
return {
ok: false,
status: 400,
message: 'Check the highlighted fields.',
errors: parsed.errors
};
const req = parsed.value;
let taskId: number | null = null;
let taskUpdatedAt: string | null = null;
let payload: Record<string, unknown> = {};
if (req.kind === 'add') {
if (!(await store.categoryExists(req.fields!.category)))
return {
ok: false,
status: 400,
message: 'Check the highlighted fields.',
errors: { category: 'Pick a category' }
};
payload = { ...req.fields! };
} else {
const found = await store.getTask(req.taskSlug);
if (!found) return { ok: false, status: 404, message: 'That task does not exist (anymore).' };
taskId = found.task.id;
taskUpdatedAt = found.task.updatedAt;
if (req.kind === 'edit') {
const change = changedFields(store.currentFields(found.task), req.fields as TaskFields);
if (!Object.keys(change).length)
return { ok: false, status: 400, message: 'Nothing changed: edit at least one field.' };
if (change.category && !(await store.categoryExists(change.category)))
return {
ok: false,
status: 400,
message: 'Check the highlighted fields.',
errors: { category: 'Pick a category' }
};
payload = change as Record<string, unknown>;
}
}
const wait = limiter.hit(opts.ip);
if (wait !== null)
return {
ok: false,
status: 429,
message: `You've sent a lot of requests. Try again in ${Math.ceil(wait / 60)} min.`,
retryAfter: wait
};
if ((await store.pendingTotal()) >= MAX_PENDING)
return {
ok: false,
status: 503,
message: 'The review queue is full right now. Please try again in a day or two.',
retryAfter: 86_400
};
if (!(await verifyTurnstile(opts.token)))
return {
ok: false,
status: 400,
message: 'The anti-bot check failed or expired. Try it again.',
errors: { turnstile: 'Complete the check' }
};
const row = await store.insertRequest({
kind: req.kind,
taskId,
taskUpdatedAt,
payload,
reason: req.reason,
displayName: req.displayName
});
publishQueue(await store.pendingTotal());
return { ok: true, id: row.id };
}

View File

@@ -0,0 +1,555 @@
import type { TransactionSql } from 'postgres';
// jsonb values are sent as JSON text with a ::jsonb cast: drizzle swaps the shared client's
// json serializers for pass-through ones, so sql.json() would send a raw object.
import { client as sql } from '#lib/server/db/index.js';
import {
EDITABLE_FIELDS,
RESERVED_SLUGS,
changedFields,
type Category,
type PublicTask,
type RequestKind,
type RequestStatus,
type TaskFields
} from '#lib/worldtodo/shared.js';
// Postgres access for the Worldwide Todo List (schema worldtodo). Hand-written SQL on the
// shared postgres.js pool; every write that applies a request runs in one transaction.
const TASK_COLUMNS = sql`
t.id, t.slug, t.title, t.description, t.scale::text as scale, t.category,
t.location_name, t.country_code, t.lat, t.lon, t.status::text as status, t.source_urls,
t.source_date, t.confidence, t.verification_note, t.as_of::text as as_of,
t.created_at, t.updated_at, t.deleted_at`;
type Row = Record<string, unknown>;
// The shared client returns timestamptz as Postgres text ("2026-10-26 01:13:36.1+00") because
// drizzle swaps in pass-through parsers, so normalise to ISO here.
function toDate(v: unknown): Date | null {
if (v instanceof Date) return v;
if (!v) return null;
const d = new Date(
String(v)
.replace(' ', 'T')
.replace(/([+-]\d\d)$/, '$1:00')
);
return Number.isNaN(d.getTime()) ? null : d;
}
const iso = (v: unknown) => toDate(v)?.toISOString() ?? '';
export function toTask(r: Row): PublicTask {
return {
id: Number(r.id),
slug: String(r.slug),
title: String(r.title),
description: String(r.description),
scale: r.scale as PublicTask['scale'],
category: String(r.category),
locationName: String(r.location_name),
countryCode: (r.country_code as string | null) ?? null,
lat: r.lat === null || r.lat === undefined ? null : Number(r.lat),
lon: r.lon === null || r.lon === undefined ? null : Number(r.lon),
status: r.status as PublicTask['status'],
sourceUrls: (r.source_urls as string[]) ?? [],
sourceDate: (r.source_date as string | null) ?? null,
confidence: r.confidence === 'medium' ? 'medium' : 'high',
verificationNote: (r.verification_note as string | null) ?? null,
asOf: String(r.as_of),
createdAt: iso(r.created_at),
updatedAt: iso(r.updated_at),
...(r.pending !== undefined ? { pending: Number(r.pending) } : {})
};
}
export async function listCategories(): Promise<Category[]> {
const rows = await sql`select slug, label, icon, colour from worldtodo.categories order by label`;
return rows.map((r) => ({
slug: r.slug,
label: r.label,
icon: r.icon,
colour: r.colour
}));
}
export type Sort = 'updated' | 'scale' | 'title';
export type ListFilter = {
scale?: string;
category?: string;
/** 'todo' (open + in progress, the default), 'open', 'in_progress', 'done' or 'all'. */
status?: string;
q?: string;
};
function likeEscape(s: string) {
return s.replace(/[\\%_]/g, (c) => `\\${c}`);
}
function whereFor(f: ListFilter, opts: { skipScale?: boolean } = {}) {
const parts = [sql`t.deleted_at is null`];
if (f.scale && !opts.skipScale) parts.push(sql`t.scale = ${f.scale}::worldtodo.scale`);
if (f.category) parts.push(sql`t.category = ${f.category}`);
const status = f.status || 'todo';
if (status === 'todo') parts.push(sql`t.status <> 'done'`);
else if (status !== 'all') parts.push(sql`t.status = ${status}::worldtodo.status`);
const q = (f.q ?? '').trim().slice(0, 100);
if (q) {
const like = `%${likeEscape(q)}%`;
parts.push(
sql`(t.search @@ websearch_to_tsquery('simple', ${q}) or t.title ilike ${like}
or t.location_name ilike ${like} or t.description ilike ${like})`
);
}
return parts.reduce((acc, p, i) => (i === 0 ? p : sql`${acc} and ${p}`));
}
const SCALE_ORDER = sql`array_position(array['planet','region','country','city','street']::worldtodo.scale[], t.scale)`;
function orderFor(sort: Sort) {
if (sort === 'title') return sql`lower(t.title) asc, t.id asc`;
if (sort === 'scale') return sql`${SCALE_ORDER} asc, lower(t.title) asc, t.id asc`;
return sql`t.updated_at desc, t.id desc`;
}
function cursorKey(sort: Sort, r: Row): unknown[] {
if (sort === 'title') return [String(r.title).toLowerCase(), Number(r.id)];
if (sort === 'scale') return [Number(r.scale_pos), String(r.title).toLowerCase(), Number(r.id)];
return [iso(r.updated_at), Number(r.id)];
}
function afterCursor(sort: Sort, cursor: string | undefined) {
if (!cursor) return null;
try {
const k = JSON.parse(Buffer.from(cursor, 'base64url').toString()) as unknown[];
if (sort === 'title' && k.length === 2)
return sql`(lower(t.title), t.id) > (${String(k[0])}, ${Number(k[1])})`;
if (sort === 'scale' && k.length === 3)
return sql`(${SCALE_ORDER}, lower(t.title), t.id) > (${Number(k[0])}, ${String(k[1])}, ${Number(k[2])})`;
if (sort === 'updated' && k.length === 2)
return sql`(t.updated_at, t.id) < (${String(k[0])}::timestamptz, ${Number(k[1])})`;
} catch {
/* bad cursor: start from the top */
}
return null;
}
/** Paged list (keyset cursor), excludes deleted tasks; each task carries its pending count. */
export async function listTasks(
f: ListFilter,
sort: Sort = 'updated',
limit = 50,
cursor?: string
): Promise<{ tasks: PublicTask[]; nextCursor: string | null }> {
const after = afterCursor(sort, cursor);
const where = after ? sql`${whereFor(f)} and ${after}` : whereFor(f);
const rows = await sql`
select ${TASK_COLUMNS}, ${SCALE_ORDER} as scale_pos,
(select count(*) from worldtodo.change_requests r
where r.task_id = t.id and r.status = 'pending')::int as pending
from worldtodo.tasks t
where ${where}
order by ${orderFor(sort)}
limit ${limit + 1}`;
const more = rows.length > limit;
const page = rows.slice(0, limit);
return {
tasks: page.map(toTask),
nextCursor:
more && page.length
? Buffer.from(JSON.stringify(cursorKey(sort, page.at(-1)!))).toString('base64url')
: null
};
}
/** Counts for the hero (per status) and the scale strip (per scale, other filters applied). */
export async function counts(f: ListFilter) {
const [byStatus, byScale] = await Promise.all([
sql`select status::text as status, count(*)::int as n from worldtodo.tasks
where deleted_at is null group by status`,
sql`select t.scale::text as scale, count(*)::int as n from worldtodo.tasks t
where ${whereFor(f, { skipScale: true })} group by t.scale`
]);
const status: Record<string, number> = { open: 0, in_progress: 0, done: 0 };
for (const r of byStatus) status[r.status] = r.n;
const scale: Record<string, number> = {};
for (const r of byScale) scale[r.scale] = r.n;
return { status, scale };
}
export async function getTask(slug: string, includeDeleted = false) {
const [row] = await sql`
select ${TASK_COLUMNS},
(select count(*) from worldtodo.change_requests r
where r.task_id = t.id and r.status = 'pending')::int as pending
from worldtodo.tasks t
where t.slug = ${slug} ${includeDeleted ? sql`` : sql`and t.deleted_at is null`}`;
if (!row) return null;
return { task: toTask(row), deletedAt: row.deleted_at ? iso(row.deleted_at) : null };
}
/** Summary of a request that is safe to show publicly: never the reason or display name. */
export function publicSummary(kind: RequestKind, payload: Record<string, unknown> | null) {
if (kind === 'delete') return 'Request to remove this task';
if (kind === 'add') return 'New task';
const keys = Object.keys(payload ?? {}).filter((k) =>
(EDITABLE_FIELDS as readonly string[]).includes(k)
);
const names = keys.map((k) => k.replace('_urls', 's').replace('_', ' '));
return names.length ? `Changes ${names.join(', ')}` : 'Edit';
}
export async function taskActivity(taskId: number) {
const rows = await sql`
select id, kind::text as kind, status::text as status, payload, applied, owner_note,
created_at, decided_at
from worldtodo.change_requests
where task_id = ${taskId} and status in ('approved', 'pending')
order by coalesce(decided_at, created_at) desc
limit 100`;
const history = rows
.filter((r) => r.status === 'approved')
.map((r) => ({
id: Number(r.id),
kind: r.kind as RequestKind,
summary: publicSummary(r.kind, (r.applied ?? r.payload) as Record<string, unknown>),
ownerNote: (r.owner_note as string | null) ?? null,
at: iso(r.decided_at ?? r.created_at)
}));
const pending = rows
.filter((r) => r.status === 'pending')
.map((r) => ({
id: Number(r.id),
kind: r.kind as RequestKind,
summary: publicSummary(r.kind, r.payload as Record<string, unknown>),
at: iso(r.created_at)
}));
return { history, pending };
}
/** The add request that created a task (if any), for the "added" line in its history. */
export async function pendingTotal(): Promise<number> {
const [r] =
await sql`select count(*)::int as n from worldtodo.change_requests where status = 'pending'`;
return r.n;
}
export async function categoryExists(slug: string) {
const [r] = await sql`select 1 from worldtodo.categories where slug = ${slug}`;
return !!r;
}
export async function insertRequest(input: {
kind: RequestKind;
taskId: number | null;
taskUpdatedAt: string | null;
payload: Record<string, unknown>;
reason: string;
displayName: string | null;
}) {
const [r] = await sql`
insert into worldtodo.change_requests
(kind, task_id, task_updated_at, payload, reason, display_name)
values (${input.kind}::worldtodo.req_kind, ${input.taskId}, ${input.taskUpdatedAt},
${JSON.stringify(input.payload)}::jsonb, ${input.reason}, ${input.displayName})
returning id, created_at`;
return { id: Number(r.id), createdAt: iso(r.created_at) };
}
export async function requestStatus(id: number) {
const [r] = await sql`
select r.id, r.kind::text as kind, r.status::text as status, r.created_at, r.decided_at,
r.owner_note, t.slug, t.title, t.deleted_at
from worldtodo.change_requests r
left join worldtodo.tasks t on t.id = coalesce(r.task_id, (r.applied->>'_task_id')::bigint)
where r.id = ${id}`;
if (!r) return null;
return {
id: Number(r.id),
kind: r.kind as RequestKind,
status: r.status as RequestStatus,
createdAt: iso(r.created_at),
decidedAt: r.decided_at ? iso(r.decided_at) : null,
ownerNote: r.status === 'pending' ? null : ((r.owner_note as string | null) ?? null),
// Only link a task the public can already see (an approved add, or the edited task).
task:
r.slug && !r.deleted_at && (r.kind !== 'add' || r.status === 'approved')
? { slug: String(r.slug), title: String(r.title) }
: null
};
}
export type QueueItem = {
id: number;
kind: RequestKind;
status: RequestStatus;
payload: Record<string, unknown>;
applied: Record<string, unknown> | null;
reason: string;
displayName: string | null;
ownerNote: string | null;
createdAt: string;
decidedAt: string | null;
task: PublicTask | null;
taskDeleted: boolean;
stale: boolean;
similar: { slug: string; title: string; similarity: number }[];
};
async function similarTitles(title: string, excludeId: number | null) {
if (!title) return [];
try {
const rows = await sql`
select slug, title, public.similarity(title, ${title}) as s
from worldtodo.tasks
where deleted_at is null and public.similarity(title, ${title}) > 0.4
${excludeId ? sql`and id <> ${excludeId}` : sql``}
order by s desc limit 3`;
return rows.map((r) => ({
slug: String(r.slug),
title: String(r.title),
similarity: Number(r.s)
}));
} catch {
return []; // pg_trgm is optional
}
}
export async function queue(status: RequestStatus, limit = 100): Promise<QueueItem[]> {
const order = status === 'pending' ? sql`r.created_at asc` : sql`r.decided_at desc nulls last`;
// Request columns are aliased: TASK_COLUMNS brings its own id, status and created_at.
const rows = await sql`
select r.id as req_id, r.kind::text as kind, r.status::text as req_status, r.payload,
r.applied, r.reason, r.display_name, r.owner_note, r.created_at as req_created_at,
r.decided_at, r.task_updated_at, ${TASK_COLUMNS}, t.id as t_id
from worldtodo.change_requests r
left join worldtodo.tasks t on t.id = coalesce(r.task_id, (r.applied->>'_task_id')::bigint)
where r.status = ${status}::worldtodo.req_status
order by ${order}
limit ${limit}`;
return Promise.all(
rows.map(async (r) => {
const task = r.t_id ? toTask({ ...r, id: r.t_id }) : null;
const deleted = !!r.deleted_at;
const stale =
r.kind !== 'add' &&
(deleted ||
!!(
r.task_updated_at &&
r.updated_at &&
(toDate(r.updated_at)?.getTime() ?? 0) > (toDate(r.task_updated_at)?.getTime() ?? 0) + 1
));
const payload = (r.payload ?? {}) as Record<string, unknown>;
const title = String(payload.title ?? '');
return {
id: Number(r.req_id),
kind: r.kind as RequestKind,
status: r.req_status as RequestStatus,
payload,
applied: (r.applied as Record<string, unknown> | null) ?? null,
reason: String(r.reason),
displayName: (r.display_name as string | null) ?? null,
ownerNote: (r.owner_note as string | null) ?? null,
createdAt: iso(r.req_created_at),
decidedAt: r.decided_at ? iso(r.decided_at) : null,
task,
taskDeleted: deleted,
stale: status === 'pending' && stale,
similar:
status === 'pending' && title && r.kind !== 'delete'
? await similarTitles(title, task?.id ?? null)
: []
} satisfies QueueItem;
})
);
}
export class DecisionError extends Error {
constructor(
message: string,
public status = 400
) {
super(message);
}
}
function slugBase(title: string) {
return (
title
.toLowerCase()
.normalize('NFKD')
.replace(/[\u0300-\u036f]/g, '')
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 70)
.replace(/-+$/g, '') || 'task'
);
}
// eslint-disable-next-line @typescript-eslint/no-empty-object-type
type Tx = TransactionSql<{}>;
async function uniqueTaskSlug(tx: Tx, title: string) {
const base = slugBase(title);
let slug = RESERVED_SLUGS.has(base) ? `${base}-2` : base;
let n = RESERVED_SLUGS.has(base) ? 3 : 2;
while ((await tx`select 1 from worldtodo.tasks where slug = ${slug}`).length)
slug = `${base}-${n++}`;
return slug;
}
/** Full field set for an add (payload) with defaults. */
function addFields(p: Record<string, unknown>): TaskFields {
return {
title: String(p.title ?? ''),
description: String(p.description ?? ''),
scale: p.scale as TaskFields['scale'],
category: String(p.category ?? ''),
location_name: String(p.location_name ?? ''),
country_code: (p.country_code as string | null) ?? null,
lat: (p.lat as number | null) ?? null,
lon: (p.lon as number | null) ?? null,
status: (p.status as TaskFields['status']) ?? 'open',
source_urls: (p.source_urls as string[]) ?? []
};
}
const today = () => new Date().toISOString().slice(0, 10);
export type Applied = { type: 'task.added' | 'task.updated' | 'task.deleted'; slug: string };
/**
* Approve a pending request: apply the add, edit or delete, set updated_at, mark the request
* approved with the optional note, all in one transaction. `tweaked` (edit & approve) replaces
* the payload: for an add the full field set, for an edit the proposed field set (diffed here).
*/
export async function approve(
id: number,
note: string | null,
tweaked: TaskFields | null
): Promise<Applied> {
return sql.begin(async (tx) => {
const [req] = await tx`
select id, kind::text as kind, status::text as status, task_id, payload
from worldtodo.change_requests where id = ${id} for update`;
if (!req) throw new DecisionError('Request not found', 404);
if (req.status !== 'pending') throw new DecisionError('This request was already decided');
const payload = (req.payload ?? {}) as Record<string, unknown>;
if (req.kind === 'add') {
const fields = tweaked ?? addFields(payload);
const [cat] = await tx`select 1 from worldtodo.categories where slug = ${fields.category}`;
if (!cat) throw new DecisionError('Unknown category');
const slug = await uniqueTaskSlug(tx, fields.title);
const [task] = await tx`
insert into worldtodo.tasks (slug, title, description, scale, category, location_name,
country_code, lat, lon, status, source_urls, as_of, confidence)
values (${slug}, ${fields.title}, ${fields.description}, ${fields.scale}::worldtodo.scale,
${fields.category}, ${fields.location_name}, ${fields.country_code}, ${fields.lat},
${fields.lon}, ${fields.status}::worldtodo.status, ${fields.source_urls},
${today()}, 'medium')
returning id`;
await tx`
update worldtodo.change_requests
set status = 'approved', decided_at = now(), owner_note = ${note},
applied = ${JSON.stringify({ ...fields, _task_id: Number(task.id) })}::jsonb
where id = ${id}`;
return { type: 'task.added', slug } satisfies Applied;
}
const [row] = await tx`
select ${TASK_COLUMNS} from worldtodo.tasks t where t.id = ${req.task_id} for update`;
if (!row) throw new DecisionError('The task no longer exists', 409);
const current = toTask(row);
if (req.kind === 'delete') {
if (!row.deleted_at)
await tx`update worldtodo.tasks set deleted_at = now(), updated_at = now() where id = ${current.id}`;
await tx`
update worldtodo.change_requests
set status = 'approved', decided_at = now(), owner_note = ${note}, applied = '{}'::jsonb
where id = ${id}`;
return { type: 'task.deleted', slug: current.slug } satisfies Applied;
}
// edit
if (row.deleted_at) throw new DecisionError('The task was removed; reject this edit', 409);
const proposed = tweaked ?? { ...currentFields(current), ...pick(payload) };
const change = changedFields(currentFields(current), proposed);
if (!Object.keys(change).length) throw new DecisionError('Nothing would change');
if (change.category) {
const [cat] = await tx`select 1 from worldtodo.categories where slug = ${change.category}`;
if (!cat) throw new DecisionError('Unknown category');
}
const next = { ...currentFields(current), ...change };
await tx`
update worldtodo.tasks set
title = ${next.title}, description = ${next.description},
scale = ${next.scale}::worldtodo.scale, category = ${next.category},
location_name = ${next.location_name}, country_code = ${next.country_code},
lat = ${next.lat}, lon = ${next.lon}, status = ${next.status}::worldtodo.status,
source_urls = ${next.source_urls}, as_of = ${today()}, updated_at = now()
where id = ${current.id}`;
await tx`
update worldtodo.change_requests
set status = 'approved', decided_at = now(), owner_note = ${note},
applied = ${JSON.stringify(change)}::jsonb
where id = ${id}`;
return { type: 'task.updated', slug: current.slug } satisfies Applied;
}) as Promise<Applied>;
}
export async function reject(id: number, note: string | null) {
const rows = await sql`
update worldtodo.change_requests
set status = 'rejected', decided_at = now(), owner_note = ${note}
where id = ${id} and status = 'pending'
returning id`;
if (!rows.length) {
const [r] =
await sql`select status::text as status from worldtodo.change_requests where id = ${id}`;
throw new DecisionError(
r ? 'This request was already decided' : 'Request not found',
r ? 400 : 404
);
}
}
/** Direct owner edit: applied at once and logged as an auto-approved request. */
export async function ownerEdit(slug: string, proposed: TaskFields, note: string | null) {
const found = await getTask(slug);
if (!found) throw new DecisionError('Task not found', 404);
const change = changedFields(currentFields(found.task), proposed);
if (!Object.keys(change).length) throw new DecisionError('Nothing would change');
const req = await insertRequest({
kind: 'edit',
taskId: found.task.id,
taskUpdatedAt: found.task.updatedAt,
payload: change as Record<string, unknown>,
reason: 'Direct edit by the owner.',
displayName: null
});
return approve(req.id, note ?? 'Direct owner edit', null);
}
export function currentFields(t: PublicTask): TaskFields {
return {
title: t.title,
description: t.description,
scale: t.scale,
category: t.category,
location_name: t.locationName,
country_code: t.countryCode,
lat: t.lat,
lon: t.lon,
status: t.status,
source_urls: t.sourceUrls
};
}
/** Whitelist payload keys to the editable fields. */
export function pick(p: Record<string, unknown>): Partial<TaskFields> {
const out: Record<string, unknown> = {};
for (const k of EDITABLE_FIELDS) if (k in p) out[k] = p[k];
return out as Partial<TaskFields>;
}
export async function sitemapSlugs() {
const rows =
await sql`select slug, updated_at from worldtodo.tasks where deleted_at is null order by id`;
return rows.map((r) => ({ slug: String(r.slug), updatedAt: iso(r.updated_at) }));
}

View File

@@ -0,0 +1,32 @@
import { TURNSTILE_SECRET_KEY } from '$app/env/private';
import { PUBLIC_TURNSTILE_SITE_KEY } from '$app/env/public';
// Cloudflare Turnstile on the request form, checked server-side through Siteverify.
// Widget "stanrunge.dev worldwide todo list" (managed mode, domain stanrunge.dev).
// Without both keys, request submission is switched off and the form says so.
const SITEVERIFY = 'https://challenges.cloudflare.com/turnstile/v0/siteverify';
export const turnstileSiteKey = () => PUBLIC_TURNSTILE_SITE_KEY || '';
export const turnstileConfigured = () => !!(TURNSTILE_SECRET_KEY && PUBLIC_TURNSTILE_SITE_KEY);
export async function verifyTurnstile(
token: string,
fetcher: typeof fetch = fetch
): Promise<boolean> {
if (!turnstileConfigured() || !token || token.length > 2048) return false;
try {
const body = new URLSearchParams({ secret: TURNSTILE_SECRET_KEY, response: token });
const res = await fetcher(SITEVERIFY, {
method: 'POST',
body,
signal: AbortSignal.timeout(8000)
});
if (!res.ok) return false;
const data = (await res.json()) as { success?: boolean };
return data.success === true;
} catch (e) {
console.error('[worldtodo] turnstile siteverify failed', (e as Error).message);
return false;
}
}

View File

@@ -0,0 +1,126 @@
import { describe, expect, it } from 'vitest';
import { looksLikeBot, normaliseUrl, parseRequest, plain } from './validate';
const add = {
kind: 'add',
title: 'Fix the <b>broken</b> streetlights on Main Street',
description: 'Half the lights have been out since August.\n\n<script>x</script>Please fix.',
scale: 'street',
category: 'safety',
location_name: 'Main Street, Gouda',
country_code: 'nl',
lat: '52.01',
lon: '4.71',
status: 'open',
source_urls: ['https://example.org/news', '', 'https://example.org/news'],
reason: 'I walk here every evening and it is dark.',
display_name: ''
};
describe('worldtodo request validation', () => {
it('accepts a valid add, strips HTML and normalises', () => {
const r = parseRequest(add);
expect(r.ok).toBe(true);
if (!r.ok) return;
expect(r.value.fields?.title).toBe('Fix the broken streetlights on Main Street');
expect(r.value.fields?.description).not.toContain('<');
expect(r.value.fields?.country_code).toBe('NL');
expect(r.value.fields?.lat).toBe(52.01);
expect(r.value.fields?.source_urls).toEqual(['https://example.org/news']);
expect(r.value.displayName).toBeNull();
});
it('requires sources for add and edit, and a reason of 10+ chars', () => {
const r = parseRequest({ ...add, source_urls: [], reason: 'short' });
expect(r.ok).toBe(false);
if (r.ok) return;
expect(r.errors.source_urls).toBeTruthy();
expect(r.errors.reason).toBeTruthy();
});
it('rejects non-http URLs, too many URLs and overlong URLs', () => {
expect(normaliseUrl('javascript:alert(1)')).toBeNull();
expect(normaliseUrl('ftp://example.org/x')).toBeNull();
expect(normaliseUrl(`https://example.org/${'a'.repeat(500)}`)).toBeNull();
expect(normaliseUrl('https://example.org/ok')).toBe('https://example.org/ok');
const many = Array.from({ length: 6 }, (_, i) => `https://e${i}.org/`);
const r = parseRequest({ ...add, source_urls: many });
expect(r.ok).toBe(false);
});
it('needs both coordinates or neither, in range', () => {
expect(parseRequest({ ...add, lon: '' }).ok).toBe(false);
expect(parseRequest({ ...add, lat: '91' }).ok).toBe(false);
expect(parseRequest({ ...add, lat: '', lon: '' }).ok).toBe(true);
});
it('delete needs only a task and a reason', () => {
const r = parseRequest({
kind: 'delete',
task: 'some-task',
reason: 'It was fixed last week.'
});
expect(r.ok).toBe(true);
if (r.ok) expect(r.value.fields).toBeNull();
expect(parseRequest({ kind: 'delete', reason: 'It was fixed last week.' }).ok).toBe(false);
});
it('caps the display name at 40 characters', () => {
expect(parseRequest({ ...add, display_name: 'x'.repeat(41) }).ok).toBe(false);
const r = parseRequest({ ...add, display_name: ' <i>Anna</i> ' });
expect(r.ok && r.value.displayName).toBe('Anna');
});
it('flags the honeypot and too-fast submissions', () => {
const now = 1_000_000;
expect(looksLikeBot('', now - 10_000, now)).toBe(false);
expect(looksLikeBot('http://spam', now - 10_000, now)).toBe(true);
expect(looksLikeBot('', now - 1000, now)).toBe(true);
expect(looksLikeBot('', 'garbage', now)).toBe(true);
});
it('plain() keeps paragraphs but drops control characters', () => {
expect(plain('a\u0000b\r\n\r\n\r\nc', true)).toBe('a b\n\nc');
expect(plain(' one\ntwo ')).toBe('one two');
});
it('add without the optional keys defaults them (JSON API)', () => {
const { country_code, lat, lon, status, display_name, ...minimal } = add;
void [country_code, lat, lon, status, display_name];
const r = parseRequest(minimal);
expect(r.ok).toBe(true);
if (!r.ok) return;
expect(r.value.fields).toMatchObject({
country_code: null,
lat: null,
lon: null,
status: 'open'
});
});
it('a partial edit is validated on top of the current task', () => {
const base = {
title: 'Fix the broken streetlights on Main Street',
description: 'Out since August.',
scale: 'street',
category: 'safety',
location_name: 'Main Street, Gouda',
country_code: 'NL',
lat: 52.01,
lon: 4.71,
status: 'open',
source_urls: ['https://example.org/news']
};
const r = parseRequest(
{
kind: 'edit',
task: 'main-street',
status: 'in_progress',
reason: 'Works started on Monday.'
},
base
);
expect(r.ok).toBe(true);
if (r.ok) expect(r.value.fields).toMatchObject({ ...base, status: 'in_progress' });
});
});

View File

@@ -0,0 +1,247 @@
import { z } from 'zod';
import {
WORLDTODO_REQUEST_KINDS,
WORLDTODO_SCALES,
WORLDTODO_STATUSES,
type RequestKind,
type TaskFields
} from '#lib/worldtodo/shared.js';
// Validation for public change requests (form action and JSON API share it). Everything is
// plain text: HTML tags and control characters are stripped, Markdown is never rendered.
// Source URLs must be http(s) and are stored as given (never fetched on submit).
export const MAX_URLS = 5;
export const MAX_URL_LENGTH = 500;
/** Minimum time between opening the form and submitting it (cheap bot filter). */
export const MIN_FILL_MS = 3000;
/** Plain text: no tags, no control characters, trimmed. `multiline` keeps line breaks. */
export function plain(value: unknown, multiline = false): string {
if (typeof value !== 'string') return '';
let s = value.replace(/<[^>]*>?/g, '').replace(/[<>]/g, '');
s = s.replace(/\r\n?/g, '\n');
// eslint-disable-next-line no-control-regex
s = s.replace(multiline ? /[\u0000-\u0009\u000b-\u001f\u007f]/g : /[\u0000-\u001f\u007f]/g, ' ');
s = multiline
? s
.split('\n')
.map((l) => l.replace(/[ \t]+/g, ' ').trim())
.join('\n')
.replace(/\n{3,}/g, '\n\n')
: s.replace(/\s+/g, ' ');
return s.trim();
}
export function normaliseUrl(raw: string): string | null {
const s = raw.trim();
if (!s || s.length > MAX_URL_LENGTH) return null;
try {
const u = new URL(s);
if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;
if (!u.hostname.includes('.')) return null;
return s;
} catch {
return null;
}
}
const optionalNumber = (min: number, max: number, label: string) =>
z.union([z.string(), z.number(), z.null(), z.undefined()]).transform((v, ctx) => {
if (v === null || v === undefined || (typeof v === 'string' && v.trim() === '')) return null;
const n = typeof v === 'number' ? v : Number(v);
if (!Number.isFinite(n) || n < min || n > max) {
ctx.addIssue({ code: 'custom', message: `${label} must be between ${min} and ${max}` });
return z.NEVER;
}
return Math.round(n * 1e6) / 1e6;
});
export const fieldsSchema = z
.object({
title: z
.string({ message: 'Add a title' })
.transform((v) => plain(v))
.pipe(
z
.string()
.min(5, 'Title needs at least 5 characters')
.max(140, 'Title can be at most 140 characters')
),
description: z
.string({ message: 'Add a description' })
.transform((v) => plain(v, true))
.pipe(
z
.string()
.min(10, 'Description needs at least 10 characters')
.max(2000, 'Description can be at most 2000 characters')
),
scale: z.enum(WORLDTODO_SCALES, { message: 'Pick a scale' }),
category: z
.string({ message: 'Pick a category' })
.trim()
.regex(/^[a-z][a-z0-9-]{1,30}$/, 'Pick a category'),
location_name: z
.string({ message: 'Add a location' })
.transform((v) => plain(v))
.pipe(
z
.string()
.min(2, 'Location needs at least 2 characters')
.max(120, 'Location can be at most 120 characters')
),
country_code: z.union([z.string(), z.null(), z.undefined()]).transform((v, ctx) => {
const s = (v ?? '').trim().toUpperCase();
if (!s) return null;
if (!/^[A-Z]{2}$/.test(s)) {
ctx.addIssue({ code: 'custom', message: 'Country code is two letters, e.g. NL' });
return z.NEVER;
}
return s;
}),
lat: optionalNumber(-90, 90, 'Latitude'),
lon: optionalNumber(-180, 180, 'Longitude'),
status: z.enum(WORLDTODO_STATUSES, { message: 'Pick a status' }),
source_urls: z.array(z.string()).transform((list, ctx) => {
const filled = list.map((s) => s.trim()).filter(Boolean);
if (filled.length > MAX_URLS) {
ctx.addIssue({ code: 'custom', message: `At most ${MAX_URLS} source links` });
return z.NEVER;
}
const out: string[] = [];
for (const raw of filled) {
const url = normaliseUrl(raw);
if (!url) {
ctx.addIssue({
code: 'custom',
message: `Source links must be http(s) URLs of at most ${MAX_URL_LENGTH} characters`
});
return z.NEVER;
}
if (!out.includes(url)) out.push(url);
}
if (!out.length) {
ctx.addIssue({ code: 'custom', message: 'Add at least one source link' });
return z.NEVER;
}
return out;
})
})
.superRefine((v, ctx) => {
if ((v.lat === null) !== (v.lon === null))
ctx.addIssue({ code: 'custom', path: ['lat'], message: 'Give both latitude and longitude' });
});
const metaSchema = z.object({
kind: z.enum(WORLDTODO_REQUEST_KINDS, { message: 'Unknown request type' }),
task: z.string().trim().max(120).optional().default(''),
reason: z
.string()
.transform((v) => plain(v, true))
.pipe(
z
.string()
.min(10, 'Reason needs at least 10 characters')
.max(1000, 'Reason can be at most 1000 characters')
),
display_name: z
.union([z.string(), z.null(), z.undefined()])
.transform((v) => plain(v ?? '').slice(0, 200))
.pipe(z.string().max(40, 'Display name can be at most 40 characters'))
.transform((v) => v || null)
});
export type ParsedRequest = {
kind: RequestKind;
taskSlug: string;
fields: TaskFields | null;
reason: string;
displayName: string | null;
};
export type FieldErrors = Record<string, string>;
/** Raw input (from FormData or JSON) → a validated request, or field errors. */
export function parseRequest(
input: Record<string, unknown>,
base?: Record<string, unknown>
): { ok: true; value: ParsedRequest } | { ok: false; errors: FieldErrors } {
const errors: FieldErrors = {};
// Optional keys are defaulted here: zod 4 treats a missing key in front of a transform as
// required.
const meta = metaSchema.safeParse({
...input,
task: input.task ?? '',
display_name: input.display_name ?? ''
});
if (!meta.success)
for (const issue of meta.error.issues)
errors[String(issue.path[0] ?? 'form')] ??= issue.message;
const kind = meta.success ? meta.data.kind : (input.kind as RequestKind);
if ((kind === 'edit' || kind === 'delete') && !(meta.success && meta.data.task))
errors.task ??= 'Which task is this about?';
let fields: TaskFields | null = null;
if (kind === 'add' || kind === 'edit') {
// Edits may send only the fields that change: the rest comes from the current task.
// Optional keys are defaulted for the same zod 4 reason as above.
const merged = { ...(kind === 'edit' ? base : undefined), ...input };
const urls = merged.source_urls;
const parsed = fieldsSchema.safeParse({
...merged,
country_code: merged.country_code ?? '',
lat: merged.lat ?? '',
lon: merged.lon ?? '',
status: merged.status ?? (kind === 'add' ? 'open' : undefined),
source_urls: Array.isArray(urls) ? urls : typeof urls === 'string' ? [urls] : []
});
if (parsed.success) fields = parsed.data;
else
for (const issue of parsed.error.issues)
errors[String(issue.path[0] ?? 'form')] ??= issue.message;
}
if (Object.keys(errors).length || !meta.success) return { ok: false, errors };
return {
ok: true,
value: {
kind: meta.data.kind,
taskSlug: meta.data.task,
fields,
reason: meta.data.reason,
displayName: meta.data.display_name
}
};
}
/** FormData → the raw input record parseRequest takes. */
export function formInput(form: FormData): Record<string, unknown> {
const out: Record<string, unknown> = {};
for (const key of [
'kind',
'task',
'title',
'description',
'scale',
'category',
'location_name',
'country_code',
'lat',
'lon',
'status',
'reason',
'display_name'
]) {
const v = form.get(key);
if (typeof v === 'string') out[key] = v;
}
out.source_urls = form.getAll('source_urls').filter((v): v is string => typeof v === 'string');
return out;
}
/** Honeypot filled in, or submitted faster than a person could. */
export function looksLikeBot(honeypot: unknown, startedAt: unknown, now = Date.now()): boolean {
if (typeof honeypot === 'string' && honeypot.trim()) return true;
const started = Number(startedAt);
if (!Number.isFinite(started) || started <= 0) return true;
return now - started < MIN_FILL_MS;
}