website/scripts/stats/wikipolls.ts

// A16 Wikipedia poll tables (CC BY-SA 4.0). For each series we fetch the rendered page via the
// MediaWiki API, keep the wikitext snapshot per revision (stats.op_wiki_snapshot), and upsert
// one row per poll and option into stats.op_poll. Polls dropped from the page stay in the DB.
import { getJson, insertChunks, log, run, sql } from './common';
import { fieldworkEnd, findTables, firstNumber, tableGrid, type Cell } from './htmltable';

const API = 'https://en.wikipedia.org/w/api.php';

type Series = {
	series: string;
	page: string;
	/** Header labels that identify the table(s) to read. */
	match: string[];
	/** How many matching tables to read (newest first on these pages). */
	tables: number;
	pollster: RegExp;
	date: RegExp;
	sample?: RegExp;
	skip: RegExp;
};

export const WIKI_SERIES: Series[] = [
	{
		series: 'nl-tk-seats',
		page: 'Opinion_polling_for_the_next_Dutch_general_election',
		match: ['Polling firm', 'Fieldwork date'],
		tables: 2,
		pollster: /^Polling firm/i,
		date: /^Fieldwork/i,
		sample: /^Sample/i,
		skip: /^(Lead|Ref|Ref\.)$/i
	},
	{
		series: 'us-trump-approval-aggregate',
		page: 'Opinion_polling_on_the_second_Trump_presidency',
		match: ['Aggregator', 'Approve'],
		tables: 1,
		pollster: /^Aggregator/i,
		date: /^Updated/i,
		skip: /^(Lead|Net)$/i
	},
	{
		series: 'us-trump-approval',
		page: 'Opinion_polling_on_the_second_Trump_presidency',
		match: ['Poll source', 'Approve'],
		tables: 13,
		pollster: /^Poll source/i,
		date: /^Date/i,
		sample: /^Sample/i,
		skip: /^(MoE|Net|Lead)$/i
	},
	{
		series: 'kr-lee-approval',
		page: 'Opinion_polling_on_the_Lee_Jae_Myung_presidency',
		match: ['Polling firm', 'Approve'],
		tables: 2,
		pollster: /^Polling firm/i,
		date: /^Fieldwork/i,
		sample: /^Sample/i,
		skip: /^(Margin|Net|Lead)/i
	},
	{
		series: 'kr-party',
		page: 'Opinion_polling_for_the_2028_South_Korean_legislative_election',
		match: ['Polling firm', 'DPK'],
		tables: 2,
		pollster: /^Polling firm/i,
		date: /^Fieldwork/i,
		sample: /^Sample/i,
		skip: /^(Margin|Lead)/i
	}
];

type Poll = {
	series: string;
	poll_key: string;
	fieldwork_end: string;
	pollster: string;
	sample: number | null;
	option: string;
	value: number;
	seen_at: Date;
};

export function pollsFromHtml(
	s: Series,
	html: string,
	today = new Date()
): Omit<Poll, 'seen_at'>[] {
	const out: Omit<Poll, 'seen_at'>[] = [];
	const tables = findTables(html)
		.map((t) => ({
			g: tableGrid(t.html),
			year: Number(t.heading.match(/\b(19|20)\d{2}\b/)?.[0] ?? today.getFullYear())
		}))
		.filter(({ g }) => {
			const head = (g[0] ?? []).map((c) => c.text);
			return s.match.every((m) => head.some((h) => h.startsWith(m)));
		})
		.slice(0, s.tables);
	const limit = new Date(today.getTime() + 7 * 864e5).toISOString().slice(0, 10);
	for (const { g, year } of tables) {
		const head = g[0].map((c) => c.text.replace(/\s*\/\s*/g, '/'));
		const col = (re: RegExp) => head.findIndex((h) => re.test(h));
		const [cPoll, cDate, cSample] = [col(s.pollster), col(s.date), s.sample ? col(s.sample) : -1];
		const options = head
			.map((h, i) => ({ h, i }))
			.filter(
				({ h, i }) =>
					i !== cPoll &&
					i !== cDate &&
					i !== cSample &&
					h &&
					!s.skip.test(h) &&
					head.indexOf(h) === i
			);
		for (const row of g.slice(1)) {
			if (row.every((c) => c.header) || row.length < head.length - 2) continue;
			const dateCell: Cell | undefined = row[cDate];
			if (!dateCell) continue;
			let end = fieldworkEnd(dateCell.text, dateCell.sort, year);
			if (end && end > limit) end = `${Number(end.slice(0, 4)) - 1}${end.slice(4)}`;
			if (!end) continue;
			const pollster = row[cPoll]?.text ?? '';
			if (!pollster || pollster === dateCell.text) continue;
			const sample = cSample >= 0 ? firstNumber(row[cSample]?.text ?? '') : null;
			const key = `${end}|${pollster}|${sample ?? ''}`;
			for (const { h, i } of options) {
				const v = firstNumber(row[i]?.text ?? '');
				if (v != null && !row[i]?.header)
					out.push({
						series: s.series,
						poll_key: key,
						fieldwork_end: end,
						pollster,
						sample,
						option: h,
						value: v
					});
			}
		}
	}
	// Same poll twice (repeated across tables): keep the first.
	const seen = new Set<string>();
	return out.filter((p) => {
		const k = `${p.poll_key}|${p.option}`;
		if (seen.has(k)) return false;
		seen.add(k);
		return true;
	});
}

export async function wikipolls(opts: { job: string }) {
	await run('wikipedia', opts.job, async (note) => {
		const pages = new Map<string, { revid: number; html: string; wikitext: string }>();
		let rows = 0;
		for (const s of WIKI_SERIES) {
			let page = pages.get(s.page);
			if (!page) {
				const url = `${API}?action=parse&page=${encodeURIComponent(s.page)}&prop=text|wikitext|revid&redirects=1&format=json&formatversion=2`;
				const res = await getJson<{
					parse?: { revid: number; text: string; wikitext: string };
					error?: { info: string };
				}>(url);
				if (!res.parse) {
					note(`${s.page}: ${res.error?.info ?? 'no parse result'}`);
					continue;
				}
				page = { revid: res.parse.revid, html: res.parse.text, wikitext: res.parse.wikitext };
				pages.set(s.page, page);
				await sql`
					insert into stats.op_wiki_snapshot (page, revid, fetched_at, wikitext)
					values (${s.page}, ${page.revid}, now(), ${page.wikitext})
					on conflict (page, revid) do nothing`;
			}
			const polls = pollsFromHtml(s, page.html);
			if (!polls.length) {
				note(`${s.series}: no polls parsed from ${s.page} rev ${page.revid}`);
				continue;
			}
			const now = new Date();
			await sql.begin(async (tx) => {
				await insertChunks(
					tx,
					'op_poll',
					polls.map((p) => ({ ...p, seen_at: now })),
					'on conflict (series, poll_key, option) do update set value = excluded.value, seen_at = excluded.seen_at'
				);
			});
			log(`wikipolls ${s.series}: ${polls.length} values`);
			rows += polls.length;
		}
		return rows;
	});
}