stats step 3: elections, party positions and opinion feeds (schema, collectors, timers); world-opinions data moved into stats.wo_*

scripts/stats/wikipolls.ts

@@ -0,0 +1,195 @@
+// A16 Wikipedia poll tables (CC BY-SA 4.0). For each series we fetch the rendered page via the
+// MediaWiki API, keep the wikitext snapshot per revision (stats.op_wiki_snapshot), and upsert
+// one row per poll and option into stats.op_poll. Polls dropped from the page stay in the DB.
+import { getJson, insertChunks, log, run, sql } from './common';
+import { fieldworkEnd, findTables, firstNumber, tableGrid, type Cell } from './htmltable';
+
+const API = 'https://en.wikipedia.org/w/api.php';
+
+type Series = {
+	series: string;
+	page: string;
+	/** Header labels that identify the table(s) to read. */
+	match: string[];
+	/** How many matching tables to read (newest first on these pages). */
+	tables: number;
+	pollster: RegExp;
+	date: RegExp;
+	sample?: RegExp;
+	skip: RegExp;
+};
+
+export const WIKI_SERIES: Series[] = [
+	{
+		series: 'nl-tk-seats',
+		page: 'Opinion_polling_for_the_next_Dutch_general_election',
+		match: ['Polling firm', 'Fieldwork date'],
+		tables: 2,
+		pollster: /^Polling firm/i,
+		date: /^Fieldwork/i,
+		sample: /^Sample/i,
+		skip: /^(Lead|Ref|Ref\.)$/i
+	},
+	{
+		series: 'us-trump-approval-aggregate',
+		page: 'Opinion_polling_on_the_second_Trump_presidency',
+		match: ['Aggregator', 'Approve'],
+		tables: 1,
+		pollster: /^Aggregator/i,
+		date: /^Updated/i,
+		skip: /^(Lead|Net)$/i
+	},
+	{
+		series: 'us-trump-approval',
+		page: 'Opinion_polling_on_the_second_Trump_presidency',
+		match: ['Poll source', 'Approve'],
+		tables: 13,
+		pollster: /^Poll source/i,
+		date: /^Date/i,
+		sample: /^Sample/i,
+		skip: /^(MoE|Net|Lead)$/i
+	},
+	{
+		series: 'kr-lee-approval',
+		page: 'Opinion_polling_on_the_Lee_Jae_Myung_presidency',
+		match: ['Polling firm', 'Approve'],
+		tables: 2,
+		pollster: /^Polling firm/i,
+		date: /^Fieldwork/i,
+		sample: /^Sample/i,
+		skip: /^(Margin|Net|Lead)/i
+	},
+	{
+		series: 'kr-party',
+		page: 'Opinion_polling_for_the_2028_South_Korean_legislative_election',
+		match: ['Polling firm', 'DPK'],
+		tables: 2,
+		pollster: /^Polling firm/i,
+		date: /^Fieldwork/i,
+		sample: /^Sample/i,
+		skip: /^(Margin|Lead)/i
+	}
+];
+
+type Poll = {
+	series: string;
+	poll_key: string;
+	fieldwork_end: string;
+	pollster: string;
+	sample: number | null;
+	option: string;
+	value: number;
+	seen_at: Date;
+};
+
+export function pollsFromHtml(
+	s: Series,
+	html: string,
+	today = new Date()
+): Omit<Poll, 'seen_at'>[] {
+	const out: Omit<Poll, 'seen_at'>[] = [];
+	const tables = findTables(html)
+		.map((t) => ({
+			g: tableGrid(t.html),
+			year: Number(t.heading.match(/\b(19|20)\d{2}\b/)?.[0] ?? today.getFullYear())
+		}))
+		.filter(({ g }) => {
+			const head = (g[0] ?? []).map((c) => c.text);
+			return s.match.every((m) => head.some((h) => h.startsWith(m)));
+		})
+		.slice(0, s.tables);
+	const limit = new Date(today.getTime() + 7 * 864e5).toISOString().slice(0, 10);
+	for (const { g, year } of tables) {
+		const head = g[0].map((c) => c.text.replace(/\s*\/\s*/g, '/'));
+		const col = (re: RegExp) => head.findIndex((h) => re.test(h));
+		const [cPoll, cDate, cSample] = [col(s.pollster), col(s.date), s.sample ? col(s.sample) : -1];
+		const options = head
+			.map((h, i) => ({ h, i }))
+			.filter(
+				({ h, i }) =>
+					i !== cPoll &&
+					i !== cDate &&
+					i !== cSample &&
+					h &&
+					!s.skip.test(h) &&
+					head.indexOf(h) === i
+			);
+		for (const row of g.slice(1)) {
+			if (row.every((c) => c.header) || row.length < head.length - 2) continue;
+			const dateCell: Cell | undefined = row[cDate];
+			if (!dateCell) continue;
+			let end = fieldworkEnd(dateCell.text, dateCell.sort, year);
+			if (end && end > limit) end = `${Number(end.slice(0, 4)) - 1}${end.slice(4)}`;
+			if (!end) continue;
+			const pollster = row[cPoll]?.text ?? '';
+			if (!pollster || pollster === dateCell.text) continue;
+			const sample = cSample >= 0 ? firstNumber(row[cSample]?.text ?? '') : null;
+			const key = `${end}|${pollster}|${sample ?? ''}`;
+			for (const { h, i } of options) {
+				const v = firstNumber(row[i]?.text ?? '');
+				if (v != null && !row[i]?.header)
+					out.push({
+						series: s.series,
+						poll_key: key,
+						fieldwork_end: end,
+						pollster,
+						sample,
+						option: h,
+						value: v
+					});
+			}
+		}
+	}
+	// Same poll twice (repeated across tables): keep the first.
+	const seen = new Set<string>();
+	return out.filter((p) => {
+		const k = `${p.poll_key}|${p.option}`;
+		if (seen.has(k)) return false;
+		seen.add(k);
+		return true;
+	});
+}
+
+export async function wikipolls(opts: { job: string }) {
+	await run('wikipedia', opts.job, async (note) => {
+		const pages = new Map<string, { revid: number; html: string; wikitext: string }>();
+		let rows = 0;
+		for (const s of WIKI_SERIES) {
+			let page = pages.get(s.page);
+			if (!page) {
+				const url = `${API}?action=parse&page=${encodeURIComponent(s.page)}&prop=text|wikitext|revid&redirects=1&format=json&formatversion=2`;
+				const res = await getJson<{
+					parse?: { revid: number; text: string; wikitext: string };
+					error?: { info: string };
+				}>(url);
+				if (!res.parse) {
+					note(`${s.page}: ${res.error?.info ?? 'no parse result'}`);
+					continue;
+				}
+				page = { revid: res.parse.revid, html: res.parse.text, wikitext: res.parse.wikitext };
+				pages.set(s.page, page);
+				await sql`
+					insert into stats.op_wiki_snapshot (page, revid, fetched_at, wikitext)
+					values (${s.page}, ${page.revid}, now(), ${page.wikitext})
+					on conflict (page, revid) do nothing`;
+			}
+			const polls = pollsFromHtml(s, page.html);
+			if (!polls.length) {
+				note(`${s.series}: no polls parsed from ${s.page} rev ${page.revid}`);
+				continue;
+			}
+			const now = new Date();
+			await sql.begin(async (tx) => {
+				await insertChunks(
+					tx,
+					'op_poll',
+					polls.map((p) => ({ ...p, seen_at: now })),
+					'on conflict (series, poll_key, option) do update set value = excluded.value, seen_at = excluded.seen_at'
+				);
+			});
+			log(`wikipolls ${s.series}: ${polls.length} values`);
+			rows += polls.length;
+		}
+		return rows;
+	});
+}