stats step 3: elections, party positions and opinion feeds (schema, collectors, timers); world-opinions data moved into stats.wo_*

scripts/stats/htmltable.ts

@@ -0,0 +1,129 @@
+// Minimal HTML table reader for MediaWiki output: expands rowspan/colspan into a grid of cell
+// texts (references, hidden spans and styles removed). Keeps each cell's data-sort-value.
+
+export type Cell = { text: string; sort: string | null; header: boolean };
+
+const ENTITIES: Record<string, string> = {
+	amp: '&',
+	lt: '<',
+	gt: '>',
+	quot: '"',
+	apos: "'",
+	nbsp: ' ',
+	ndash: '–',
+	mdash: '—',
+	minus: '−'
+};
+export const decode = (s: string) =>
+	s.replace(/&(#x?[0-9a-f]+|\w+);/gi, (m, e: string) => {
+		if (e[0] === '#')
+			return String.fromCodePoint(
+				e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1))
+			);
+		return ENTITIES[e.toLowerCase()] ?? m;
+	});
+
+export function cellText(html: string) {
+	return decode(
+		html
+			.replace(/<sup[\s\S]*?<\/sup>/gi, '')
+			.replace(/<style[\s\S]*?<\/style>/gi, '')
+			.replace(/<span[^>]*display:\s*none[^>]*>[\s\S]*?<\/span>/gi, '')
+			.replace(/<br\s*\/?>/gi, ' ')
+			.replace(/<[^>]+>/g, '')
+	)
+		.replace(/\s+/g, ' ')
+		.trim();
+}
+
+/** All tables (outer HTML) whose opening tag matches `cls`, with the text of the nearest
+ * preceding heading. Nested tables are skipped over. */
+export function findTables(html: string, cls = /wikitable/) {
+	const out: { html: string; heading: string }[] = [];
+	const re = /<table\b[^>]*>/gi;
+	let m: RegExpExecArray | null;
+	while ((m = re.exec(html))) {
+		if (!cls.test(m[0])) continue;
+		let depth = 1;
+		const tag = /<\/?table\b[^>]*>/gi;
+		tag.lastIndex = m.index + m[0].length;
+		let t: RegExpExecArray | null;
+		while (depth && (t = tag.exec(html))) depth += t[0][1] === '/' ? -1 : 1;
+		const end = tag.lastIndex;
+		const headings = [...html.slice(0, m.index).matchAll(/<h[2-4][^>]*>([\s\S]*?)<\/h[2-4]>/gi)];
+		out.push({ html: html.slice(m.index, end), heading: cellText(headings.at(-1)?.[1] ?? '') });
+		re.lastIndex = end;
+	}
+	return out;
+}
+
+export function tableGrid(table: string): Cell[][] {
+	const grid: Cell[][] = [];
+	const pending: (Cell & { left: number })[] = []; // by column: rowspans carried down
+	const rows = table.split(/<tr\b[^>]*>/i).slice(1);
+	for (const rowHtml of rows) {
+		const row: Cell[] = [];
+		let col = 0;
+		const place = () => {
+			while (pending[col] && pending[col].left > 0) {
+				row[col] = pending[col];
+				pending[col].left--;
+				col++;
+			}
+		};
+		for (const c of rowHtml.matchAll(/<(t[hd])\b([^>]*)>([\s\S]*?)(?=<t[hd]\b|<\/tr>|$)/gi)) {
+			place();
+			const attrs = c[2];
+			const rowspan = Number(attrs.match(/rowspan="?(\d+)/i)?.[1] ?? 1);
+			const colspan = Number(attrs.match(/colspan="?(\d+)/i)?.[1] ?? 1);
+			const cell: Cell = {
+				text: cellText(c[3].replace(/<\/t[hd]>[\s\S]*$/i, '')),
+				sort: attrs.match(/data-sort-value="([^"]*)"/i)?.[1] ?? null,
+				header: c[1].toLowerCase() === 'th'
+			};
+			for (let k = 0; k < colspan; k++) {
+				row[col] = cell;
+				if (rowspan > 1) pending[col] = { ...cell, left: rowspan - 1 };
+				else delete pending[col];
+				col++;
+			}
+		}
+		place();
+		if (row.length) grid.push(row);
+	}
+	return grid;
+}
+
+const MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
+
+/** End date of a fieldwork range like "13–15 Oct 2026", "Sep 28 – Oct 2, 2026", "2026-10-15". */
+export function fieldworkEnd(
+	text: string,
+	sort: string | null,
+	fallbackYear?: number
+): string | null {
+	for (const s of [sort, text]) {
+		const iso = s?.match(/(\d{4})-(\d{1,2})-(\d{1,2})(?!.*\d{4}-\d{1,2}-\d{1,2})/);
+		if (iso) return `${iso[1]}-${iso[2].padStart(2, '0')}-${iso[3].padStart(2, '0')}`;
+	}
+	const t = text.toLowerCase().replace(/[–—−]/g, '-');
+	const year = Number(t.match(/(\d{4})(?!.*\d{4})/)?.[1] ?? fallbackYear);
+	if (!year) return null;
+	const months = [...t.matchAll(/\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?/g)];
+	if (!months.length) return null;
+	const last = months[months.length - 1];
+	const month = MONTHS.indexOf(last[1]) + 1;
+	// Day: the number right after the last month ("Oct 2") or right before it ("15 Oct").
+	const after = t
+		.slice(last.index! + last[0].length)
+		.match(/^\s*(\d{1,2})(?:\s*-\s*(\d{1,2}))?\b(?!\d)/);
+	const before = t.slice(0, last.index).match(/(\d{1,2})\s*$/);
+	const day = Number(after?.[2] ?? after?.[1] ?? before?.[1] ?? 0);
+	if (!day || day > 31) return null;
+	return `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
+}
+
+export const firstNumber = (s: string) => {
+	const m = s.replace(/,(?=\d{3}\b)/g, '').match(/-?\d+(?:\.\d+)?/);
+	return m ? Number(m[0]) : null;
+};