stan/website
1
0
Fork 0
Code Issues 13 Pull Requests Actions Packages Projects Releases Wiki Activity

stats step 3: elections, party positions and opinion feeds (schema, collectors, timers); world-opinions data moved into stats.wo_*
All checks were successful
Deploy production / deploy (push) Successful in 1m12s
Details

Browse Source
This commit is contained in:
Stan Runge
2026-10-25 22:16:25 +00:00
parent 030f3c08ee
commit 42c9351ebd
37 changed files with 75721 additions and 892 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
scripts/stats/common.ts
View File

@@ -37,7 +37,17 @@ const hostGapMs: Record<string, number> = {
'cuatro.sim-cdn.nl': 1000,
'sdmx.ilo.org': 3000,
'unstats.un.org': 1500,
'comtradeapi.un.org': 2500
'comtradeapi.un.org': 2500,
'data.overheid.nl': 500,
'dataverse.harvard.edu': 1000,
'github.com': 1000,
'partyfacts.herokuapp.com': 2000,
'en.wikipedia.org': 1000,
'data.europa.eu': 1000,
'webgate.ec.europa.eu': 2000,
'files.worldhappiness.report': 1000,
'trends.google.com': 3000,
'api.gdeltproject.org': 5500
};
const lastHit = new Map<string, number>();
@@ -54,7 +64,7 @@ export async function http(url: string, init: RequestInit = {}, tries = 4): Prom
response = await fetch(url, {
...init,
headers: { 'user-agent': USER_AGENT, accept: '*/*', ...(init.headers ?? {}) },
signal: AbortSignal.timeout(180_000)
signal: init.signal ?? AbortSignal.timeout(180_000)
});
if (response.ok) return response;
if (response.status !== 429 && response.status < 500) {
@@ -248,6 +258,74 @@ export const SOURCES = [
licence: 'Free use with attribution (UN Comtrade public preview)',
key_required: false,
attribution: 'Source: UN Comtrade Database (United Nations Statistics Division)'
},
{
source_id: 'kiesraad',
name: 'Kiesraad election results (data.overheid.nl)',
base_url: 'https://data.overheid.nl/dataset?q=verkiezingsuitslag',
licence: 'CC0 1.0',
key_required: false,
attribution: 'Source: Kiesraad, verkiezingsuitslagen (data.overheid.nl), CC0'
},
{
source_id: 'mitelection',
name: 'MIT Election Data and Science Lab, U.S. President 1976–2024',
base_url: 'https://doi.org/10.7910/DVN/42MVDX',
licence: 'CC0 1.0',
key_required: false,
attribution:
'Source: MIT Election Data and Science Lab, U.S. President 1976–2024 (Harvard Dataverse), CC0'
},
{
source_id: 'ches',
name: 'Chapel Hill Expert Survey (trend file) + V-Party + Party Facts',
base_url: 'https://www.chesdata.eu',
licence:
'Free for non-commercial research use with citation (CHES); V-Dem CC BY-SA 4.0; Party Facts CC BY-SA 4.0',
key_required: false,
attribution:
'Sources: Chapel Hill Expert Survey 1999–2024 (Jolly et al.); V-Dem V-Party v2 (Lührmann et al.); Party Facts (Döring & Regel)'
},
{
source_id: 'wikipedia',
name: 'Wikipedia opinion-polling tables',
base_url: 'https://en.wikipedia.org',
licence: 'CC BY-SA 4.0',
key_required: false,
attribution:
'Source: Wikipedia opinion-polling articles and their cited pollsters, CC BY-SA 4.0'
},
{
source_id: 'eurobarometer',
name: 'Standard Eurobarometer (volume A tables)',
base_url: 'https://data.europa.eu',
licence: 'Reuse authorised with acknowledgement (Commission Decision 2011/833/EU)',
key_required: false,
attribution: 'Source: European Commission, Standard Eurobarometer, © European Union'
},
{
source_id: 'whr',
name: 'World Happiness Report (Figure 2.1 data)',
base_url: 'https://worldhappiness.report',
licence: 'Free reuse with attribution',
key_required: false,
attribution: 'Source: World Happiness Report (Wellbeing Research Centre, Gallup World Poll)'
},
{
source_id: 'gtrends',
name: 'Google Trends, trending now (RSS)',
base_url: 'https://trends.google.com/trending/rss',
licence: 'Google Terms of Service; shown with attribution',
key_required: false,
attribution: 'Source: Google Trends'
},
{
source_id: 'gdelt',
name: 'GDELT Project DOC 2.0 API',
base_url: 'https://api.gdeltproject.org/api/v2/doc/doc',
licence: 'Open, unrestricted use with citation',
key_required: false,
attribution: 'Source: The GDELT Project'
}
] as const;

543
scripts/stats/elections.ts Normal file
View File

@@ -0,0 +1,543 @@
// A14 elections: NL Tweede Kamer from the Kiesraad open data (data.overheid.nl, CC0) and CBS
// 37278 (1918–2006), US presidential results by state from the MIT Election Lab (CC0).
// - Kiesraad: per election the "CSV formaat" zip (votes per list per municipality and per
// polling station; 2023+ also turnout and seats) plus the EML zips, read with HTTP ranges, for
// the official result (seats, election date) and, before 2023, the national total count.
// Polling stations are kept for Gouda (GM0513) only.
// - Results are final, so an election already loaded is skipped unless --force.
import { insertChunks, getJson, http, log, run, sql } from './common';
import { parseCsv } from './owid';
import { RemoteZip } from './remotezip';
const CKAN =
'https://data.overheid.nl/data/api/3/action/package_search?q=verkiezingsuitslag%20Tweede%20Kamer&rows=100';
const STATION_MUNIS = new Set(['GM0513']);
type Result = {
election_id: string;
level: string;
area_code: string;
area_name: string | null;
party: string;
votes: number | null;
seats: number | null;
};
type Turnout = {
election_id: string;
level: string;
area_code: string;
area_name: string | null;
electorate: number | null;
ballots: number | null;
valid: number | null;
blank: number | null;
invalid: number | null;
};
type Election = {
election_id: string;
country: string;
kind: string;
date: string;
name: string;
source_id: string;
total_seats: number | null;
};
async function save(e: Election, results: Result[], turnout: Turnout[]) {
// Merge duplicate keys (a list name can repeat within a municipality in old files).
const merged = new Map<string, Result>();
for (const r of results) {
const key = `${r.level}|${r.area_code}|${r.party}`;
const prev = merged.get(key);
if (!prev) merged.set(key, { ...r });
else {
prev.votes =
prev.votes == null && r.votes == null ? null : (prev.votes ?? 0) + (r.votes ?? 0);
prev.seats = prev.seats ?? r.seats;
}
}
await sql.begin(async (tx) => {
await tx`delete from stats.elec_result where election_id = ${e.election_id}`;
await tx`delete from stats.elec_turnout where election_id = ${e.election_id}`;
await tx`delete from stats.elec_election where election_id = ${e.election_id}`;
await insertChunks(tx, 'elec_election', [e]);
await insertChunks(tx, 'elec_result', [...merged.values()]);
await insertChunks(tx, 'elec_turnout', turnout);
});
return merged.size + turnout.length;
}
const num = (s: string | undefined) => {
if (s == null || s.trim() === '') return null;
const n = Number(s.replace(/\s/g, ''));
return Number.isFinite(n) ? n : null;
};
const gm = (code: string) => `GM${code.replace(/^\D+/, '').padStart(4, '0')}`;
const xmlText = (s: string) =>
s
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&amp;/g, '&');
/** Rows of a semicolon CSV as objects keyed by the header. */
function rowsOf(text: string): Record<string, string>[] {
const [header, ...rows] = parseCsv(text, ';');
return rows
.filter((r) => r.length >= header.length - 1 && r.some((x) => x !== ''))
.map((r) => Object.fromEntries(header.map((h, i) => [h.trim(), (r[i] ?? '').trim()])));
}
/** EML 520 (Resultaat): election date and seats per list (elected candidates per affiliation). */
function parseResultaat(xml: string) {
const date = xml.match(/<kr:ElectionDate>([\d-]+)<\/kr:ElectionDate>/)?.[1] ?? null;
const seats = new Map<string, number>();
let current = '';
for (const m of xml.matchAll(/<Selection>([\s\S]*?)<\/Selection>/g)) {
const aff = m[1].match(/<RegisteredName>([\s\S]*?)<\/RegisteredName>/);
if (aff) {
current = xmlText(aff[1].trim());
if (!seats.has(current)) seats.set(current, 0);
} else if (/<Candidate>/.test(m[1]) && /<Elected>yes<\/Elected>/.test(m[1]))
seats.set(current, (seats.get(current) ?? 0) + 1);
}
return { date, seats };
}
/** EML 510d (Totaaltelling): national votes per list and the count totals. */
function parseTotaaltelling(xml: string) {
const total = xml.match(/<TotalVotes>([\s\S]*?)<\/TotalVotes>/)?.[1] ?? '';
const votes = new Map<string, number>();
for (const m of total.matchAll(
/<Selection><AffiliationIdentifier[^>]*><RegisteredName>([\s\S]*?)<\/RegisteredName><\/AffiliationIdentifier><ValidVotes>(\d+)<\/ValidVotes>/g
))
votes.set(xmlText(m[1].trim()), Number(m[2]));
const pick = (re: RegExp) => num(total.match(re)?.[1]);
return {
votes,
electorate: pick(/<Cast>(\d+)<\/Cast>/),
ballots: pick(/<TotalCounted>(\d+)<\/TotalCounted>/),
invalid: pick(/<RejectedVotes ReasonCode="ongeldig">(\d+)</),
blank: pick(/<RejectedVotes ReasonCode="blanco">(\d+)</)
};
}
type CkanResource = { url: string; name?: string; format?: string };
type CkanPackage = { name: string; title: string; resources: CkanResource[] };
async function kiesraad(force: boolean, note: (w: string) => void) {
const res = await getJson<{ result: { results: CkanPackage[] } }>(CKAN);
const packages = res.result.results
.map((p) => ({ p, year: p.name.match(/^verkiezingsuitslag-tweede-kamer-(\d{4})$/)?.[1] }))
.filter((x): x is { p: CkanPackage; year: string } => !!x.year)
.sort((a, b) => a.year.localeCompare(b.year));
const loaded = new Set(
(await sql`select election_id from stats.elec_election where source_id = 'kiesraad'`).map(
(r) => r.election_id as string
)
);
let rows = 0;
for (const { p, year } of packages) {
const id = `NL-TK-${year}`;
if (loaded.has(id) && !force) continue;
const csvRes = p.resources.find((r) => /csv/i.test(decodeURIComponent(r.url)));
if (!csvRes) {
note(`${id}: no CSV zip`);
continue;
}
const csvZip = await RemoteZip.open(csvRes.url);
// EML zips: find the official result (and, for older years, the national count).
let resultaat: ReturnType<typeof parseResultaat> | null = null;
let totaal: ReturnType<typeof parseTotaaltelling> | null = null;
for (const r of p.resources.filter((r) => r !== csvRes && /\.zip$/i.test(r.url))) {
const zip = await RemoteZip.open(r.url);
const re = zip.find(/(^|\/)Resultaat_TK\d+[^/]*\.eml\.xml$/i)[0];
if (re && !resultaat) resultaat = parseResultaat(await zip.text(re));
const tt = zip.find(/(^|\/)Totaaltelling_TK\d+[^/]*\.eml\.xml$/i)[0];
if (tt && !totaal && !csvZip.find(/_uitslag\.csv$/i).length)
totaal = parseTotaaltelling(await zip.text(tt));
if (resultaat && (totaal || csvZip.find(/_uitslag\.csv$/i).length)) break;
}
const results: Result[] = [];
const turnout: Turnout[] = [];
const base = { election_id: id };
const uitslag = csvZip.find(/_uitslag\.csv$/i)[0];
if (uitslag) {
for (const r of rowsOf(await csvZip.text(uitslag))) {
const code = r.RegioCode ?? '';
const level = code.startsWith('L') ? 'national' : /^[GO]/.test(code) ? 'muni' : null;
if (!level) continue;
const area_code = level === 'national' ? 'NL' : gm(code);
const area_name = level === 'national' ? 'Nederland' : r.Regio;
const v = num(r.Waarde);
if (r.VeldType === 'LijstAantalStemmen' && r.LijstNaam)
results.push({
...base,
level,
area_code,
area_name,
party: r.LijstNaam,
votes: v,
seats: null
});
else if (r.VeldType === 'LijstAantalZetels' && r.LijstNaam && level === 'national')
results.push({
...base,
level,
area_code,
area_name,
party: r.LijstNaam,
votes: null,
seats: v
});
else if (
[
'Kiesgerechtigden',
'Opkomst',
'AantalGeldigeStemmen',
'AantalBlancoStemmen',
'AantalOngeldigeStemmen'
].includes(r.VeldType)
) {
let t = turnout.find((x) => x.level === level && x.area_code === area_code);
if (!t) {
t = {
...base,
level,
area_code,
area_name,
electorate: null,
ballots: null,
valid: null,
blank: null,
invalid: null
};
turnout.push(t);
}
const col = {
Kiesgerechtigden: 'electorate',
Opkomst: 'ballots',
AantalGeldigeStemmen: 'valid',
AantalBlancoStemmen: 'blank',
AantalOngeldigeStemmen: 'invalid'
}[r.VeldType] as 'electorate' | 'ballots' | 'valid' | 'blank' | 'invalid';
t[col] = v;
}
}
} else {
const gem = csvZip.find(/Per_Gemeente\.csv$/i)[0];
if (!gem) throw new Error(`${id}: no municipality CSV in ${csvRes.url}`);
for (const r of rowsOf(await csvZip.text(gem)))
results.push({
...base,
level: 'muni',
area_code: gm(r.GemeenteCode),
area_name: r.GemeenteNaam,
party: r.PartijNaam,
votes: num(r.AantalStemmen),
seats: null
});
if (totaal) {
const seats = resultaat?.seats ?? new Map<string, number>();
for (const [party, votes] of totaal.votes)
results.push({
...base,
level: 'national',
area_code: 'NL',
area_name: 'Nederland',
party,
votes,
seats: seats.get(party) ?? null
});
const valid = [...totaal.votes.values()].reduce((a, b) => a + b, 0);
turnout.push({
...base,
level: 'national',
area_code: 'NL',
area_name: 'Nederland',
electorate: totaal.electorate,
ballots: totaal.ballots,
valid,
blank: totaal.blank,
invalid: totaal.invalid
});
} else note(`${id}: no Totaaltelling EML, national total missing`);
}
// Merge votes and seats rows of the national level (2023+ lists them separately).
const nat = new Map<string, Result>();
for (const r of results.filter((r) => r.level === 'national')) {
const prev = nat.get(r.party);
if (prev) {
prev.votes ??= r.votes;
prev.seats ??= r.seats;
} else nat.set(r.party, r);
}
const finalResults = [...results.filter((r) => r.level !== 'national'), ...nat.values()];
// Polling stations (Gouda only).
const sb = csvZip.find(/Per_Stembureau\.csv$/i)[0];
if (sb) {
for (const r of rowsOf(await csvZip.text(sb))) {
const muni = gm(r.GemeenteCode);
if (!STATION_MUNIS.has(muni)) continue;
finalResults.push({
...base,
level: 'station',
area_code: `${muni}-${r.StembureauCode}`,
area_name: r.StembureauNaam || null,
party: r.PartijNaam,
votes: num(r.AantalStemmen),
seats: null
});
}
} else note(`${id}: no polling-station CSV`);
const totalSeats = [...nat.values()].reduce((a, r) => a + (r.seats ?? 0), 0) || null;
const date = resultaat?.date ?? KNOWN_DATES[year];
if (!date) throw new Error(`${id}: no election date`);
if (!resultaat) note(`${id}: no Resultaat EML (seats/date from CSV or table)`);
const n = await save(
{
election_id: id,
country: 'NLD',
kind: 'TK',
date,
name: `Tweede Kamer ${year}`,
source_id: 'kiesraad',
total_seats: totalSeats
},
finalResults,
turnout
);
log(
`kiesraad ${id}: ${n} rows (${finalResults.filter((r) => r.level === 'station').length} Gouda station rows)`
);
rows += n;
}
return rows;
}
// Election days (CBS 37278 only has the year). Source: Kiesraad / Parlement.com.
const KNOWN_DATES: Record<string, string> = {
'1918': '1918-07-03',
'1922': '1922-07-05',
'1925': '1925-07-01',
'1929': '1929-07-03',
'1933': '1933-04-26',
'1937': '1937-05-26',
'1946': '1946-05-17',
'1948': '1948-07-07',
'1952': '1952-06-25',
'1956': '1956-06-13',
'1959': '1959-03-12',
'1963': '1963-05-15',
'1967': '1967-02-15',
'1971': '1971-04-28',
'1972': '1972-11-29',
'1977': '1977-05-25',
'1981': '1981-05-26',
'1982': '1982-09-08',
'1986': '1986-05-21',
'1989': '1989-09-06',
'1994': '1994-05-03',
'1998': '1998-05-06',
'2002': '2002-05-15',
'2003': '2003-01-22',
'2006': '2006-11-22',
'2010': '2010-06-09',
'2012': '2012-09-12',
'2017': '2017-03-15',
'2021': '2021-03-17',
'2023': '2023-11-22',
'2025': '2025-10-29'
};
type CbsProp = { Key: string; Title: string; Type: string; ParentID?: number | null; ID: number };
async function cbs37278() {
const base = 'https://opendata.cbs.nl/ODataApi/odata/37278';
const props = (await getJson<{ value: CbsProp[] }>(`${base}/DataProperties`)).value;
const groups = new Map(props.filter((p) => p.Type === 'TopicGroup').map((p) => [p.ID, p.Title]));
const data = (
await getJson<{ value: Record<string, string | number | null>[] }>(`${base}/TypedDataSet`)
).value;
let rows = 0;
for (const d of data) {
const year = String(d.Perioden).slice(0, 4);
const id = `NL-TK-${year}`;
const results = new Map<string, Result>();
const turnout: Turnout = {
election_id: id,
level: 'national',
area_code: 'NL',
area_name: 'Nederland',
electorate: null,
ballots: null,
valid: null,
blank: null,
invalid: null
};
for (const p of props.filter((p) => p.Type === 'Topic')) {
const v = d[p.Key] == null ? null : Number(d[p.Key]);
const group = groups.get(p.ParentID ?? -1) ?? '';
if (p.Key.startsWith('Kiesgerechtigden')) turnout.electorate = v;
else if (p.Key.startsWith('TotaalStemmen')) turnout.ballots = v;
else if (p.Key.startsWith('OngeldigeStemmen')) turnout.invalid = v;
else if (p.Key.startsWith('TotaalGeldigeStemmen')) turnout.valid = v;
else if (/^(Opkomst|TotaalAantalZetels)/.test(p.Key)) continue;
else if (v != null && /stemmen per partij/i.test(group)) {
const r = results.get(p.Title) ?? {
election_id: id,
level: 'national',
area_code: 'NL',
area_name: 'Nederland',
party: p.Title,
votes: null,
seats: null
};
r.votes = v;
results.set(p.Title, r);
} else if (v != null && /zetel/i.test(group)) {
const r = results.get(p.Title) ?? {
election_id: id,
level: 'national',
area_code: 'NL',
area_name: 'Nederland',
party: p.Title,
votes: null,
seats: null
};
r.seats = v;
results.set(p.Title, r);
}
}
const list = [...results.values()].filter((r) => r.votes || r.seats);
const totalSeats = list.reduce((a, r) => a + (r.seats ?? 0), 0) || null;
rows += await save(
{
election_id: id,
country: 'NLD',
kind: 'TK',
date: KNOWN_DATES[year] ?? `${year}-01-01`,
name: `Tweede Kamer ${year}`,
source_id: 'cbs',
total_seats: totalSeats
},
list,
[turnout]
);
}
return rows;
}
/** US presidential election day: the Tuesday after the first Monday in November. */
function usElectionDay(year: number) {
const d = new Date(Date.UTC(year, 10, 1));
const firstMonday = 1 + ((8 - d.getUTCDay()) % 7);
return `${year}-11-${String(firstMonday + 1).padStart(2, '0')}`;
}
async function mit(note: (w: string) => void) {
const ds = await getJson<{
data: { latestVersion: { files: { dataFile: { id: number; filename: string } }[] } };
}>(
'https://dataverse.harvard.edu/api/datasets/:persistentId/?persistentId=doi:10.7910/DVN/42MVDX'
);
const file = ds.data.latestVersion.files
.map((f) => f.dataFile)
.filter((f) => /^\d{4}-\d{4}-president\.(csv|tab)$/.test(f.filename))
.sort((a, b) => b.filename.localeCompare(a.filename))[0];
if (!file) throw new Error('MIT president file not found in doi:10.7910/DVN/42MVDX');
const text = await (
await http(`https://dataverse.harvard.edu/api/access/datafile/${file.id}`)
).text();
const sep = file.filename.endsWith('.tab') ? '\t' : ',';
const [header, ...lines] = parseCsv(text, sep);
const col = (name: string) => header.indexOf(name);
const [cYear, cState, cPo, cParty, cVotes, cTotal] = [
'year',
'state',
'state_po',
'party_simplified',
'candidatevotes',
'totalvotes'
].map(col);
if ([cYear, cState, cPo, cParty, cVotes, cTotal].some((i) => i < 0))
throw new Error(`MIT columns changed: ${header.join(',')}`);
const byYear = new Map<number, { results: Map<string, Result>; turnout: Map<string, Turnout> }>();
for (const l of lines) {
if (l.length < header.length - 1) continue;
const year = Number(l[cYear]);
const id = `US-PRES-${year}`;
const y = byYear.get(year) ?? { results: new Map(), turnout: new Map() };
byYear.set(year, y);
const state = l[cPo];
const party = l[cParty] || 'OTHER';
const votes = Number(l[cVotes]) || 0;
for (const [level, area_code, area_name] of [
['state', state, l[cState]],
['national', 'US', 'United States']
]) {
const key = `${level}|${area_code}|${party}`;
const r = y.results.get(key) ?? {
election_id: id,
level,
area_code,
area_name,
party,
votes: 0,
seats: null
};
r.votes = (r.votes ?? 0) + votes;
y.results.set(key, r);
}
if (!y.turnout.has(state))
y.turnout.set(state, {
election_id: id,
level: 'state',
area_code: state,
area_name: l[cState],
electorate: null,
ballots: Number(l[cTotal]) || null,
valid: null,
blank: null,
invalid: null
});
}
let rows = 0;
for (const [year, y] of byYear) {
const states = [...y.turnout.values()];
states.push({
election_id: `US-PRES-${year}`,
level: 'national',
area_code: 'US',
area_name: 'United States',
electorate: null,
ballots: states.reduce((a, s) => a + (s.ballots ?? 0), 0),
valid: null,
blank: null,
invalid: null
});
rows += await save(
{
election_id: `US-PRES-${year}`,
country: 'USA',
kind: 'PRES',
date: usElectionDay(year),
name: `US presidential ${year}`,
source_id: 'mitelection',
total_seats: null
},
[...y.results.values()],
states
);
}
if (!byYear.size) note('MIT file had no rows');
return rows;
}
export async function elections(opts: { force: boolean; only?: string; job: string }) {
const want = (k: string) => !opts.only || opts.only.split(',').includes(k);
if (want('kiesraad')) await run('kiesraad', opts.job, (note) => kiesraad(opts.force, note));
if (want('cbs')) await run('cbs', `${opts.job}:37278`, () => cbs37278());
if (want('mit')) await run('mitelection', opts.job, (note) => mit(note));
}

135
scripts/stats/eurobarometer.ts Normal file
View File

@@ -0,0 +1,135 @@
// A16 Standard Eurobarometer, volume A (EU + candidate countries, weighted shares), via the
// data.europa.eu catalogue (© European Union, reuse under the Commission's reuse policy).
// Reads QA3 (most important issues facing our country), QA4 (… facing you personally), QA5
// (… facing the EU) and the trust batteries (QA6_x national/international institutions, QA8_x
// EU institutions; share "tend to trust"). Only waves published as xlsx; waves already loaded are
// skipped unless --force.
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { getJson, http, insertChunks, log, run, sql } from './common';
import { readXlsx, type Cell } from './xlsx';
const SEARCH =
'https://data.europa.eu/api/hub/search/search?q=standard%20eurobarometer&limit=100&filter=dataset';
type Dist = {
title?: string | Record<string, string>;
download_url?: string[];
access_url?: string[];
};
type Dataset = { title?: string | Record<string, string>; distributions?: Dist[] };
const en = (t: Dataset['title']) =>
typeof t === 'string' ? t : (t?.en ?? Object.values(t ?? {})[0] ?? '');
type Row = {
wave: string;
fieldwork_end: string | null;
topic: string;
qcode: string;
item: string;
country: string;
share: number;
};
const TOPICS: [RegExp, string][] = [
[/^QA3$/, 'mii_country'],
[/^QA4$/, 'mii_personal'],
[/^QA5$/, 'mii_eu'],
[/^QA6_\d+$/, 'trust'],
[/^QA8_\d+$/, 'trust_eu']
];
const str = (c: Cell | undefined) => (c == null ? '' : String(c)).trim();
const share = (c: Cell | undefined) => {
const n = typeof c === 'number' ? c : Number(str(c));
return str(c) !== '' && Number.isFinite(n) ? n : null;
};
export function parseVolumeA(sheets: Awaited<ReturnType<typeof readXlsx>>): Row[] {
const content = sheets.find((s) => s.name === 'Content');
if (!content) throw new Error('no Content sheet');
const wave = str(content.rows.find((r) => /^wave/i.test(str(r[0])))?.[1]);
const fw = str(content.rows.find((r) => /^fieldwork/i.test(str(r[0])))?.[1]);
const d = fw.match(/(\d{1,2})\/(\d{1,2})\/(\d{4})\s*$/);
const fieldwork_end = d ? `${d[3]}-${d[2].padStart(2, '0')}-${d[1].padStart(2, '0')}` : null;
if (!wave) throw new Error('no wave in Content sheet');
const questions = new Map<string, string>();
for (const r of content.rows) {
const sheet = str(r[0])
.match(/^"?([A-Z][A-Z0-9_]*)"?\)?$|"([A-Z][A-Z0-9_]*)"\)\s*$/)
?.slice(1)
.find(Boolean);
if (sheet) questions.set(sheet, str(r[2]));
}
const out: Row[] = [];
for (const s of sheets) {
const topic = TOPICS.find(([re]) => re.test(s.name))?.[1];
if (!topic) continue;
const h = s.rows.findIndex(
(r) => r.some((c) => str(c) === 'BE') && r.some((c) => str(c) === 'NL')
);
if (h < 0) continue;
const countries = s.rows[h].map((c) => str(c).split('\n').at(-1) ?? '');
// Answers come in pairs: a French row with counts, then an English row with shares.
for (let i = h + 2; i < s.rows.length; i++) {
const label = str(s.rows[i][1]);
const next = s.rows[i + 1];
if (!label || !next || !str(next[1])) continue;
const values = countries.map((_, c) => share(next[c]));
if (!values.some((v) => v != null && v <= 1)) continue;
const item = str(next[1]);
i++;
let name = item;
if (topic.startsWith('trust')) {
if (!/^tend to trust/i.test(item)) continue;
name = (questions.get(s.name) ?? s.name).split(':-').at(-1)?.trim() ?? s.name;
}
countries.forEach((country, c) => {
const v = values[c];
if (c >= 2 && country && v != null && v <= 1)
out.push({ wave, fieldwork_end, topic, qcode: s.name, item: name, country, share: v });
});
}
}
return out;
}
export async function eurobarometer(opts: { force: boolean; job: string }) {
await run('eurobarometer', opts.job, async (note) => {
const res = await getJson<{ result: { results: Dataset[] } }>(SEARCH);
const waves = res.result.results
.filter((d) => /^standard eurobarometer/i.test(en(d.title)))
.flatMap((d) =>
(d.distributions ?? [])
.filter((x) => /volume[ _]A\.xlsx$/i.test(en(x.title)))
.map((x) => ({ dataset: en(d.title), url: x.download_url?.[0] ?? x.access_url?.[0] }))
)
.filter((w): w is { dataset: string; url: string } => !!w.url);
const loaded = new Set(
(await sql`select distinct wave from stats.eb_item`).map((r) => r.wave as string)
);
let rows = 0;
for (const w of waves) {
const num = w.dataset.match(/(?:STD|Eurobarometer )(\d{2,3})/i)?.[1];
if (num && !opts.force && [...loaded].some((l) => l.split('.')[0] === num)) continue;
const file = join(tmpdir(), `eb-${process.pid}.xlsx`);
await Bun.write(file, await (await http(w.url)).arrayBuffer());
let items: Row[] = [];
try {
items = parseVolumeA(await readXlsx(file));
} catch (error) {
note(`${w.dataset}: ${error instanceof Error ? error.message : error}`);
}
await Bun.file(file).delete();
if (!items.length) continue;
const dedup = new Map(items.map((r) => [`${r.wave}|${r.topic}|${r.item}|${r.country}`, r]));
await sql.begin(async (tx) => {
await tx`delete from stats.eb_item where wave = ${items[0].wave}`;
await insertChunks(tx, 'eb_item', [...dedup.values()]);
});
log(`eurobarometer ${items[0].wave}: ${dedup.size} rows`);
rows += dedup.size;
}
return rows;
});
}

86
scripts/stats/gdelt.ts Normal file
View File

@@ -0,0 +1,86 @@
// A16 GDELT DOC 2.0 "timelinevol": daily share of monitored news coverage (by source country)
// that mentions a GKG theme, last 30 days, upserted nightly. GDELT asks for at most one request
// every 5 seconds, so the loop is deliberately slow (~12 minutes). GDELT data is open, cite GDELT.
import { http, insertChunks, run, sql } from './common';
export const GDELT_COUNTRIES: Record<string, string> = {
netherlands: 'NLD',
unitedstates: 'USA',
southkorea: 'KOR',
germany: 'DEU',
france: 'FRA',
unitedkingdom: 'GBR',
belgium: 'BEL',
japan: 'JPN',
india: 'IND',
brazil: 'BRA'
};
export const GDELT_THEMES = [
'ECON_INFLATION',
'ECON_HOUSING_PRICES',
'UNEMPLOYMENT',
'IMMIGRATION',
'REFUGEES',
'ENV_CLIMATECHANGE',
'TERROR',
'PROTEST',
'CORRUPTION',
'ELECTION',
'MILITARY',
'HEALTH_PANDEMIC'
];
export async function gdelt(opts: { job: string; timespan?: string }) {
await run('gdelt', opts.job, async (note) => {
let rows = 0;
const bad = new Set<string>();
for (const [name, iso3] of Object.entries(GDELT_COUNTRIES))
for (const theme of GDELT_THEMES) {
if (bad.has(theme)) continue;
const q = encodeURIComponent(`theme:${theme} sourcecountry:${name}`);
const url = `https://api.gdeltproject.org/api/v2/doc/doc?query=${q}&mode=timelinevol&format=json&timespan=${opts.timespan ?? '30d'}`;
let text: string;
try {
text = await (await http(url, {}, 3)).text();
} catch (error) {
note(`${theme}/${name}: ${error instanceof Error ? error.message : error}`);
continue;
}
let data: { timeline?: { data: { date: string; value: number }[] }[] };
try {
data = JSON.parse(text);
} catch {
// GDELT answers invalid queries with a plain-text message.
note(`${theme}: ${text.slice(0, 120)}`);
if (/theme|invalid/i.test(text)) bad.add(theme);
continue;
}
const points = (data.timeline?.[0]?.data ?? []).map((p) => ({
date: `${p.date.slice(0, 4)}-${p.date.slice(4, 6)}-${p.date.slice(6, 8)}`,
country: iso3,
theme,
vol_pct: p.value
}));
// Hourly resolution (short timespans): average per day.
const byDay = new Map<string, { sum: number; n: number; row: (typeof points)[number] }>();
for (const p of points) {
const d = byDay.get(p.date) ?? { sum: 0, n: 0, row: p };
d.sum += p.vol_pct;
d.n++;
byDay.set(p.date, d);
}
const daily = [...byDay.values()].map((d) => ({ ...d.row, vol_pct: d.sum / d.n }));
if (!daily.length) continue;
await sql.begin(async (tx) => {
await insertChunks(
tx,
'gdelt_theme_vol',
daily,
'on conflict (date, country, theme) do update set vol_pct = excluded.vol_pct'
);
});
rows += daily.length;
}
return rows;
});
}

56
scripts/stats/gtrends.ts Normal file
View File

@@ -0,0 +1,56 @@
// A16 Google Trends "trending now" RSS (daily; terms of service: display with attribution).
// Keeps each day's list per country; a re-run on the same day refreshes rank and traffic.
import { getText, insertChunks, run, sql } from './common';
import { decode } from './htmltable';
const GEOS = ['NL', 'US', 'KR'];
/** "200+", "2K+", "1M+" → lower bound as a number. */
const traffic = (s: string | null) => {
const m = s?.replace(/,/g, '').match(/([\d.]+)\s*([KM])?/i);
if (!m) return null;
return Math.round(Number(m[1]) * (m[2]?.toUpperCase() === 'M' ? 1e6 : m[2] ? 1e3 : 1));
};
const tag = (xml: string, name: string) => {
const m = xml.match(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`));
return m ? decode(m[1].replace(/^<!\[CDATA\[|\]\]>$/g, '').trim()) : null;
};
export async function gtrends(opts: { job: string }) {
await run('gtrends', opts.job, async () => {
let rows = 0;
for (const geo of GEOS) {
const xml = await getText(`https://trends.google.com/trending/rss?geo=${geo}`);
const today = new Intl.DateTimeFormat('en-CA', { timeZone: 'Europe/Amsterdam' }).format(
new Date()
);
const items = [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m, i) => {
const news = [...m[1].matchAll(/<ht:news_item>([\s\S]*?)<\/ht:news_item>/g)]
.map((n) => ({
title: tag(n[1], 'ht:news_item_title'),
source: tag(n[1], 'ht:news_item_source'),
url: tag(n[1], 'ht:news_item_url')
}))
.slice(0, 3);
const pub = tag(m[1], 'pubDate');
return {
geo,
date: today,
rank: i + 1,
query: tag(m[1], 'title') ?? '',
approx_traffic: traffic(tag(m[1], 'ht:approx_traffic')),
pub_date: pub ? new Date(pub) : null,
news: JSON.stringify(news)
};
});
const unique = [...new Map(items.filter((x) => x.query).map((x) => [x.query, x])).values()];
await sql.begin(async (tx) => {
await tx`delete from stats.gt_trending where geo = ${geo} and date = ${today}`;
await insertChunks(tx, 'gt_trending', unique);
});
rows += unique.length;
}
return rows;
});
}

129
scripts/stats/htmltable.ts Normal file
View File

@@ -0,0 +1,129 @@
// Minimal HTML table reader for MediaWiki output: expands rowspan/colspan into a grid of cell
// texts (references, hidden spans and styles removed). Keeps each cell's data-sort-value.
export type Cell = { text: string; sort: string | null; header: boolean };
const ENTITIES: Record<string, string> = {
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
nbsp: ' ',
ndash: '–',
mdash: '—',
minus: '−'
};
export const decode = (s: string) =>
s.replace(/&(#x?[0-9a-f]+|\w+);/gi, (m, e: string) => {
if (e[0] === '#')
return String.fromCodePoint(
e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1))
);
return ENTITIES[e.toLowerCase()] ?? m;
});
export function cellText(html: string) {
return decode(
html
.replace(/<sup[\s\S]*?<\/sup>/gi, '')
.replace(/<style[\s\S]*?<\/style>/gi, '')
.replace(/<span[^>]*display:\s*none[^>]*>[\s\S]*?<\/span>/gi, '')
.replace(/<br\s*\/?>/gi, ' ')
.replace(/<[^>]+>/g, '')
)
.replace(/\s+/g, ' ')
.trim();
}
/** All tables (outer HTML) whose opening tag matches `cls`, with the text of the nearest
* preceding heading. Nested tables are skipped over. */
export function findTables(html: string, cls = /wikitable/) {
const out: { html: string; heading: string }[] = [];
const re = /<table\b[^>]*>/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(html))) {
if (!cls.test(m[0])) continue;
let depth = 1;
const tag = /<\/?table\b[^>]*>/gi;
tag.lastIndex = m.index + m[0].length;
let t: RegExpExecArray | null;
while (depth && (t = tag.exec(html))) depth += t[0][1] === '/' ? -1 : 1;
const end = tag.lastIndex;
const headings = [...html.slice(0, m.index).matchAll(/<h[2-4][^>]*>([\s\S]*?)<\/h[2-4]>/gi)];
out.push({ html: html.slice(m.index, end), heading: cellText(headings.at(-1)?.[1] ?? '') });
re.lastIndex = end;
}
return out;
}
export function tableGrid(table: string): Cell[][] {
const grid: Cell[][] = [];
const pending: (Cell & { left: number })[] = []; // by column: rowspans carried down
const rows = table.split(/<tr\b[^>]*>/i).slice(1);
for (const rowHtml of rows) {
const row: Cell[] = [];
let col = 0;
const place = () => {
while (pending[col] && pending[col].left > 0) {
row[col] = pending[col];
pending[col].left--;
col++;
}
};
for (const c of rowHtml.matchAll(/<(t[hd])\b([^>]*)>([\s\S]*?)(?=<t[hd]\b|<\/tr>|$)/gi)) {
place();
const attrs = c[2];
const rowspan = Number(attrs.match(/rowspan="?(\d+)/i)?.[1] ?? 1);
const colspan = Number(attrs.match(/colspan="?(\d+)/i)?.[1] ?? 1);
const cell: Cell = {
text: cellText(c[3].replace(/<\/t[hd]>[\s\S]*$/i, '')),
sort: attrs.match(/data-sort-value="([^"]*)"/i)?.[1] ?? null,
header: c[1].toLowerCase() === 'th'
};
for (let k = 0; k < colspan; k++) {
row[col] = cell;
if (rowspan > 1) pending[col] = { ...cell, left: rowspan - 1 };
else delete pending[col];
col++;
}
}
place();
if (row.length) grid.push(row);
}
return grid;
}
const MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
/** End date of a fieldwork range like "13–15 Oct 2026", "Sep 28 – Oct 2, 2026", "2026-10-15". */
export function fieldworkEnd(
text: string,
sort: string | null,
fallbackYear?: number
): string | null {
for (const s of [sort, text]) {
const iso = s?.match(/(\d{4})-(\d{1,2})-(\d{1,2})(?!.*\d{4}-\d{1,2}-\d{1,2})/);
if (iso) return `${iso[1]}-${iso[2].padStart(2, '0')}-${iso[3].padStart(2, '0')}`;
}
const t = text.toLowerCase().replace(/[–—−]/g, '-');
const year = Number(t.match(/(\d{4})(?!.*\d{4})/)?.[1] ?? fallbackYear);
if (!year) return null;
const months = [...t.matchAll(/\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?/g)];
if (!months.length) return null;
const last = months[months.length - 1];
const month = MONTHS.indexOf(last[1]) + 1;
// Day: the number right after the last month ("Oct 2") or right before it ("15 Oct").
const after = t
.slice(last.index! + last[0].length)
.match(/^\s*(\d{1,2})(?:\s*-\s*(\d{1,2}))?\b(?!\d)/);
const before = t.slice(0, last.index).match(/(\d{1,2})\s*$/);
const day = Number(after?.[2] ?? after?.[1] ?? before?.[1] ?? 0);
if (!day || day > 31) return null;
return `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
}
export const firstNumber = (s: string) => {
const m = s.replace(/,(?=\d{3}\b)/g, '').match(/-?\d+(?:\.\d+)?/);
return m ? Number(m[0]) : null;
};

4
scripts/stats/owid.ts
View File

@@ -56,7 +56,7 @@ export const ENERGY_COLUMNS: Record<string, { title: string; unit: string; serie
};
/** Minimal CSV parser (quoted fields, commas inside quotes). */
export function parseCsv(text: string): string[][] {
export function parseCsv(text: string, sep = ','): string[][] {
const out: string[][] = [];
let row: string[] = [];
let field = '';
@@ -70,7 +70,7 @@ export function parseCsv(text: string): string[][] {
} else if (ch === '"') quoted = false;
else field += ch;
} else if (ch === '"') quoted = true;
else if (ch === ',') {
else if (ch === sep) {
row.push(field);
field = '';
} else if (ch === '\n' || ch === '\r') {

248
scripts/stats/parties.ts Normal file
View File

@@ -0,0 +1,248 @@
// A15 party positions: CHES trend file (party means 1999–2024) and V-Party v2 (V-Dem, via the
// vdemdata GitHub repo's RData), linked through the Party Facts crosswalk. Writes stats.wo_party
// (data-driven rows; the curated guide parties keep their own `wo:` rows) and
// stats.wo_party_position. Curated rows get a partyfacts_id when a match is unambiguous.
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { insertChunks, http, log, run, sql } from './common';
import { parseCsv } from './owid';
const CHES_URL =
'https://github.com/chesdata/chesdata.github.io/releases/download/ches-trend/1999-2024_CHES_dataset_meansV2.csv';
const VPARTY_URL = 'https://github.com/vdeminstitute/vdemdata/raw/master/data/vparty.RData';
const PARTYFACTS_URL = 'https://partyfacts.herokuapp.com/download/external-parties-csv/';
const PYTHON = process.env.STATS_PYTHON ?? `${process.env.HOME}/.local/share/stats-py/bin/python`;
const CHES_COUNTRY: Record<number, string> = {
1: 'BEL',
2: 'DNK',
3: 'DEU',
4: 'GRC',
5: 'ESP',
6: 'FRA',
7: 'IRL',
8: 'ITA',
10: 'NLD',
11: 'GBR',
12: 'PRT',
13: 'AUT',
14: 'FIN',
16: 'SWE',
20: 'BGR',
21: 'CZE',
22: 'EST',
23: 'HUN',
24: 'LVA',
25: 'LTU',
26: 'POL',
27: 'ROU',
28: 'SVK',
29: 'SVN',
31: 'HRV',
37: 'MLT',
38: 'LUX',
40: 'CYP',
9: 'NOR',
35: 'NOR',
36: 'CHE',
34: 'TUR',
45: 'ISL'
};
const CHES_FAMILY: Record<number, string> = {
1: 'radical right',
2: 'conservative',
3: 'liberal',
4: 'christian democratic',
5: 'socialist',
6: 'radical left',
7: 'green',
8: 'regionalist',
9: 'no family',
10: 'confessional',
11: 'agrarian/centre'
};
const CHES_DIMS = ['lrgen', 'lrecon', 'galtan', 'eu_position', 'immigrate_policy'];
const VPARTY_DIMS = [
'v2xpa_popul',
'v2xpa_antiplural',
'v2pariglef',
'v2paimmig',
'v2palgbt',
'v2parelig',
'v2pawelf',
'v2paculsup'
];
type Party = {
party_id: string;
iso3: string | null;
name: string | null;
abbrev: string | null;
family: string | null;
ches_id: number | null;
manifesto_id: number | null;
vparty_id: number | null;
partyfacts_id: number | null;
sort: null;
};
type Position = {
party_id: string;
year: number;
source: string;
dim: string;
value: number | null;
vote_share: number | null;
};
const table = (text: string) => {
const [header, ...rows] = parseCsv(text);
return rows
.filter((r) => r.length >= header.length - 1)
.map((r) => Object.fromEntries(header.map((h, i) => [h, r[i] ?? ''])));
};
const num = (s: string | undefined) =>
s == null || s === '' || s === 'NA' ? null : Number.isFinite(Number(s)) ? Number(s) : null;
export async function parties(opts: { job: string }) {
await run('ches', opts.job, async (note) => {
// Party Facts: dataset ids → partyfacts ids (and countries).
const pf = table(await (await http(PARTYFACTS_URL)).text());
const link = new Map<string, { pf: number | null; country: string }>();
for (const r of pf)
if (r.dataset_key === 'ches' || r.dataset_key === 'vparty' || r.dataset_key === 'manifesto')
link.set(`${r.dataset_key}:${r.dataset_party_id}`, {
pf: num(r.partyfacts_id),
country: r.country
});
const partiesById = new Map<string, Party>();
const upsert = (p: Partial<Party> & { party_id: string }) => {
const prev = partiesById.get(p.party_id);
if (!prev)
partiesById.set(p.party_id, {
iso3: null,
name: null,
abbrev: null,
family: null,
ches_id: null,
manifesto_id: null,
vparty_id: null,
partyfacts_id: null,
sort: null,
...p
});
else
for (const [k, v] of Object.entries(p))
if (v != null && (prev as Record<string, unknown>)[k] == null)
(prev as Record<string, unknown>)[k] = v;
};
const positions = new Map<string, Position>();
const put = (p: Position) => positions.set(`${p.party_id}|${p.year}|${p.source}|${p.dim}`, p);
// CHES
const ches = table(await (await http(CHES_URL)).text());
for (const r of ches) {
const chesId = num(r.party_id);
if (chesId == null) continue;
const l = link.get(`ches:${chesId}`);
const id = l?.pf != null ? `pf:${l.pf}` : `ches:${chesId}`;
const iso3 = CHES_COUNTRY[Number(r.country)] ?? l?.country ?? null;
if (!iso3) note(`CHES country ${r.country} unmapped`);
upsert({
party_id: id,
iso3,
abbrev: r.party || null,
family: CHES_FAMILY[Number(r.family)] ?? null,
ches_id: chesId,
manifesto_id: num(r.cmp_id),
partyfacts_id: l?.pf ?? null
});
for (const dim of CHES_DIMS)
if (r[dim] !== undefined)
put({
party_id: id,
year: Number(r.year),
source: 'ches',
dim,
value: num(r[dim]),
vote_share: num(r.vote)
});
}
const chesRows = positions.size;
// V-Party (RData → CSV via the stats Python venv)
let vpartyRows = 0;
try {
const file = join(tmpdir(), `vparty-${process.pid}.RData`);
await Bun.write(file, await (await http(VPARTY_URL)).arrayBuffer());
const proc = Bun.spawn([PYTHON, join(import.meta.dir, 'vparty-csv.py'), file], {
stdout: 'pipe',
stderr: 'pipe'
});
const csv = await new Response(proc.stdout).text();
const err = await new Response(proc.stderr).text();
if ((await proc.exited) !== 0) throw new Error(`vparty-csv.py failed: ${err.slice(0, 300)}`);
await Bun.file(file).delete();
for (const r of table(csv)) {
const vpId = num(r.v2paid);
if (vpId == null) continue;
const pfId = num(r.pf_party_id) ?? link.get(`vparty:${vpId}`)?.pf ?? null;
const id = pfId != null ? `pf:${pfId}` : `vp:${vpId}`;
upsert({
party_id: id,
iso3: r.country_text_id || null,
name: r.v2paenname || null,
abbrev: r.v2pashname || null,
vparty_id: vpId,
partyfacts_id: pfId,
ches_id: num(r.CHES_ID)
});
for (const dim of VPARTY_DIMS)
if (r[dim] !== undefined && r[dim] !== 'NA') {
put({
party_id: id,
year: Number(r.year),
source: 'vparty',
dim,
value: num(r[dim]),
vote_share: num(r.v2pavote)
});
vpartyRows++;
}
}
} catch (error) {
note(`V-Party skipped: ${error instanceof Error ? error.message : error}`);
}
// Names for CHES-only parties from Party Facts.
for (const r of pf)
if (r.dataset_key === 'ches') {
const id =
num(r.partyfacts_id) != null ? `pf:${r.partyfacts_id}` : `ches:${r.dataset_party_id}`;
const p = partiesById.get(id);
if (p && !p.name) p.name = r.name_english || r.name || null;
}
const list = [...partiesById.values()];
await sql.begin(async (tx) => {
await tx`delete from stats.wo_party_position where source in ('ches', 'vparty')`;
await tx`delete from stats.wo_party where party_id not like 'wo:%'`;
await insertChunks(tx, 'wo_party', list);
await insertChunks(tx, 'wo_party_position', [...positions.values()]);
// Link curated guide parties to Party Facts where exactly one data party in the same
// country has the same abbreviation (case-insensitive).
await tx`
update stats.wo_party w set partyfacts_id = m.partyfacts_id
from (
select c.party_id, min(d.partyfacts_id) as partyfacts_id
from stats.wo_party c
join stats.wo_party d on d.iso3 = c.iso3 and d.party_id not like 'wo:%'
and lower(d.abbrev) = lower(c.abbrev) and d.partyfacts_id is not null
where c.party_id like 'wo:%'
group by c.party_id having count(distinct d.partyfacts_id) = 1
) m
where w.party_id = m.party_id`;
});
log(`parties: ${list.length} parties, ${chesRows} CHES + ${vpartyRows} V-Party positions`);
return list.length + positions.size;
});
}

123
scripts/stats/remotezip.ts Normal file
View File

@@ -0,0 +1,123 @@
// Read single entries from a remote zip with HTTP range requests, so a 25 MB archive costs only
// its central directory plus the files we need. Falls back to one full download when the server
// ignores ranges. Handles stored and deflated entries (no zip64: the archives we read are small).
import { inflateRawSync } from 'node:zlib';
import { http } from './common';
export type ZipEntry = {
name: string;
method: number;
compSize: number;
size: number;
offset: number;
};
export class RemoteZip {
private full?: Uint8Array;
private constructor(
readonly url: string,
readonly size: number,
readonly entries: ZipEntry[]
) {}
private static async range(url: string, start: number, end: number, size: number) {
const res = await http(url, {
headers: { range: `bytes=${start}-${end}` },
signal: AbortSignal.timeout(1_200_000)
});
const buf = new Uint8Array(await res.arrayBuffer());
if (res.status === 206) return { buf, whole: null as Uint8Array | null };
if (buf.length !== size) throw new Error(`short download of ${url}: ${buf.length}/${size}`);
return { buf: buf.subarray(start, end + 1), whole: buf };
}
static async open(url: string): Promise<RemoteZip> {
const head = await http(url, { method: 'HEAD' });
const size = Number(head.headers.get('content-length'));
if (!size) throw new Error(`no content-length for ${url}`);
const tailLen = Math.min(size, 66000);
const tail = await RemoteZip.range(url, size - tailLen, size - 1, size);
const t = tail.buf;
const v = new DataView(t.buffer, t.byteOffset, t.byteLength);
let eocd = -1;
for (let i = t.length - 22; i >= 0; i--)
if (v.getUint32(i, true) === 0x06054b50) {
eocd = i;
break;
}
if (eocd < 0) throw new Error(`not a zip: ${url}`);
const cdSize = v.getUint32(eocd + 12, true);
const cdOffset = v.getUint32(eocd + 16, true);
const cd = tail.whole
? tail.whole.subarray(cdOffset, cdOffset + cdSize)
: (await RemoteZip.range(url, cdOffset, cdOffset + cdSize - 1, size)).buf;
const c = new DataView(cd.buffer, cd.byteOffset, cd.byteLength);
const entries: ZipEntry[] = [];
const dec = new TextDecoder();
for (let p = 0; p + 46 <= cd.length && c.getUint32(p, true) === 0x02014b50;) {
const nameLen = c.getUint16(p + 28, true);
const extraLen = c.getUint16(p + 30, true);
const commentLen = c.getUint16(p + 32, true);
entries.push({
method: c.getUint16(p + 10, true),
compSize: c.getUint32(p + 20, true),
size: c.getUint32(p + 24, true),
offset: c.getUint32(p + 42, true),
name: dec.decode(cd.subarray(p + 46, p + 46 + nameLen))
});
p += 46 + nameLen + extraLen + commentLen;
}
const zip = new RemoteZip(url, size, entries);
if (tail.whole) zip.full = tail.whole;
return zip;
}
find(pattern: RegExp) {
return this.entries.filter((e) => pattern.test(e.name) && !e.name.endsWith('/'));
}
async read(entry: ZipEntry): Promise<Uint8Array> {
let local: Uint8Array;
if (this.full)
local = this.full.subarray(entry.offset, entry.offset + 30 + 65535 * 2 + entry.compSize);
else
local = (
await RemoteZip.range(
this.url,
entry.offset,
Math.min(
this.size - 1,
entry.offset + 30 + 1024 + entry.name.length * 4 + entry.compSize
),
this.size
)
).buf;
const v = new DataView(local.buffer, local.byteOffset, local.byteLength);
if (v.getUint32(0, true) !== 0x04034b50) throw new Error(`bad local header for ${entry.name}`);
const start = 30 + v.getUint16(26, true) + v.getUint16(28, true);
if (start + entry.compSize > local.length) {
// Unusually long local extra field: fetch exactly.
const exact = await RemoteZip.range(
this.url,
entry.offset + start,
entry.offset + start + entry.compSize - 1,
this.size
);
return this.inflate(entry, exact.buf);
}
return this.inflate(entry, local.subarray(start, start + entry.compSize));
}
private inflate(entry: ZipEntry, data: Uint8Array) {
if (entry.method === 0) return data;
if (entry.method !== 8)
throw new Error(`unsupported zip method ${entry.method} (${entry.name})`);
const out = new Uint8Array(inflateRawSync(data));
if (out.length !== entry.size) throw new Error(`size mismatch for ${entry.name}`);
return out;
}
async text(entry: ZipEntry, encoding = 'utf-8') {
return new TextDecoder(encoding).decode(await this.read(entry)).replace(/^\uFEFF/, '');
}
}

14
scripts/stats/vparty-csv.py Normal file
View File

@@ -0,0 +1,14 @@
# Convert V-Dem's vparty.RData to CSV on stdout (selected columns only). Run with the stats
# Python venv (pyreadr + pandas): ~/.local/share/stats-py/bin/python vparty-csv.py file.RData
import sys
import pyreadr
COLS = [
"v2paid", "pf_party_id", "CHES_ID", "country_text_id", "year", "v2paenname", "v2pashname",
"v2pavote", "v2paseatshare", "v2xpa_popul", "v2xpa_antiplural", "v2pariglef", "v2paimmig",
"v2palgbt", "v2parelig", "v2pawelf", "v2paculsup",
]
frames = pyreadr.read_r(sys.argv[1])
df = next(iter(frames.values()))
df[[c for c in COLS if c in df.columns]].to_csv(sys.stdout, index=False)

124
scripts/stats/whr.ts Normal file
View File

@@ -0,0 +1,124 @@
// A16 World Happiness Report: Figure 2.1 data (life evaluation, 3-year average, and the
// explained components), all years 2011–latest. Tries the newest edition first.
// © Wellbeing Research Centre / Gallup World Poll data, reuse with attribution.
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { http, insertChunks, log, run, sql } from './common';
import { readXlsx, type Cell } from './xlsx';
const NAME_ISO3: Record<string, string | null> = {
'Côte d’Ivoire': 'CIV',
"Côte d'Ivoire": 'CIV',
'Ivory Coast': 'CIV',
'DR Congo': 'COD',
'Hong Kong SAR of China': 'HKG',
'Hong Kong S.A.R. of China': 'HKG',
Kosovo: 'XKX',
'Lao PDR': 'LAO',
Laos: 'LAO',
'Republic of Korea': 'KOR',
'South Korea': 'KOR',
'Republic of Moldova': 'MDA',
'Russian Federation': 'RUS',
'State of Palestine': 'PSE',
Swaziland: 'SWZ',
Eswatini: 'SWZ',
'Taiwan Province of China': 'TWN',
Türkiye: 'TUR',
Turkiye: 'TUR',
'Viet Nam': 'VNM',
'North Cyprus': null,
'Somaliland Region': null,
Czechia: 'CZE',
'Congo (Brazzaville)': 'COG',
'Congo (Kinshasa)': 'COD'
};
const n = (c: Cell | undefined) =>
typeof c === 'number'
? c
: c == null || c === ''
? null
: Number.isFinite(Number(c))
? Number(c)
: null;
export async function whr(opts: { job: string }) {
await run('whr', opts.job, async (note) => {
const yy = new Date().getFullYear() % 100;
let file: string | null = null;
for (const e of [yy + 1, yy, yy - 1]) {
try {
const res = await http(
`https://files.worldhappiness.report/WHR${e}_Data_Figure_2.1.xlsx`,
{},
1
);
file = join(tmpdir(), `whr-${process.pid}.xlsx`);
await Bun.write(file, await res.arrayBuffer());
log(`whr: edition 20${e}`);
break;
} catch {
// try an older edition
}
}
if (!file) throw new Error('no WHR Figure 2.1 file found');
const [sheet] = await readXlsx(file);
await Bun.file(file).delete();
const [header, ...rows] = sheet.rows;
const col = (re: RegExp) => header.findIndex((h) => re.test(String(h ?? '')));
const c = {
year: col(/^Year/),
rank: col(/^Rank/),
country: col(/^Country/),
ladder: col(/^Life evaluation|^Ladder/),
lower: col(/^Lower/),
upper: col(/^Upper/),
gdp: col(/GDP/),
social: col(/Social/),
health: col(/Healthy/),
freedom: col(/Freedom/),
generosity: col(/Generosity/),
corruption: col(/corruption/i),
dystopia: col(/Dystopia/)
};
if (c.year < 0 || c.country < 0 || c.ladder < 0)
throw new Error(`WHR columns changed: ${header.join('|')}`);
const iso = new Map(
(await sql`select distinct country, iso3 from stats.owid_energy where length(iso3) = 3`).map(
(r) => [r.country as string, r.iso3 as string]
)
);
const unmapped = new Set<string>();
const out = rows
.filter((r) => n(r[c.year]) && r[c.country])
.map((r) => {
const country = String(r[c.country]);
const iso3 = country in NAME_ISO3 ? NAME_ISO3[country] : (iso.get(country) ?? null);
if (!iso3 && !(country in NAME_ISO3)) unmapped.add(country);
const v = (k: keyof typeof c) => (c[k] >= 0 ? n(r[c[k]]) : null);
return {
year: v('year')!,
country,
iso3,
rank: v('rank'),
ladder: v('ladder'),
lower: v('lower'),
upper: v('upper'),
gdp: v('gdp'),
social: v('social'),
health: v('health'),
freedom: v('freedom'),
generosity: v('generosity'),
corruption: v('corruption'),
dystopia: v('dystopia')
};
});
if (unmapped.size) note(`no ISO3 for: ${[...unmapped].join(', ')}`);
await sql.begin(async (tx) => {
await tx`delete from stats.whr_score`;
await insertChunks(tx, 'whr_score', out);
});
return out.length;
});
}

195
scripts/stats/wikipolls.ts Normal file
View File

@@ -0,0 +1,195 @@
// A16 Wikipedia poll tables (CC BY-SA 4.0). For each series we fetch the rendered page via the
// MediaWiki API, keep the wikitext snapshot per revision (stats.op_wiki_snapshot), and upsert
// one row per poll and option into stats.op_poll. Polls dropped from the page stay in the DB.
import { getJson, insertChunks, log, run, sql } from './common';
import { fieldworkEnd, findTables, firstNumber, tableGrid, type Cell } from './htmltable';
const API = 'https://en.wikipedia.org/w/api.php';
type Series = {
series: string;
page: string;
/** Header labels that identify the table(s) to read. */
match: string[];
/** How many matching tables to read (newest first on these pages). */
tables: number;
pollster: RegExp;
date: RegExp;
sample?: RegExp;
skip: RegExp;
};
export const WIKI_SERIES: Series[] = [
{
series: 'nl-tk-seats',
page: 'Opinion_polling_for_the_next_Dutch_general_election',
match: ['Polling firm', 'Fieldwork date'],
tables: 2,
pollster: /^Polling firm/i,
date: /^Fieldwork/i,
sample: /^Sample/i,
skip: /^(Lead|Ref|Ref\.)$/i
},
{
series: 'us-trump-approval-aggregate',
page: 'Opinion_polling_on_the_second_Trump_presidency',
match: ['Aggregator', 'Approve'],
tables: 1,
pollster: /^Aggregator/i,
date: /^Updated/i,
skip: /^(Lead|Net)$/i
},
{
series: 'us-trump-approval',
page: 'Opinion_polling_on_the_second_Trump_presidency',
match: ['Poll source', 'Approve'],
tables: 13,
pollster: /^Poll source/i,
date: /^Date/i,
sample: /^Sample/i,
skip: /^(MoE|Net|Lead)$/i
},
{
series: 'kr-lee-approval',
page: 'Opinion_polling_on_the_Lee_Jae_Myung_presidency',
match: ['Polling firm', 'Approve'],
tables: 2,
pollster: /^Polling firm/i,
date: /^Fieldwork/i,
sample: /^Sample/i,
skip: /^(Margin|Net|Lead)/i
},
{
series: 'kr-party',
page: 'Opinion_polling_for_the_2028_South_Korean_legislative_election',
match: ['Polling firm', 'DPK'],
tables: 2,
pollster: /^Polling firm/i,
date: /^Fieldwork/i,
sample: /^Sample/i,
skip: /^(Margin|Lead)/i
}
];
type Poll = {
series: string;
poll_key: string;
fieldwork_end: string;
pollster: string;
sample: number | null;
option: string;
value: number;
seen_at: Date;
};
export function pollsFromHtml(
s: Series,
html: string,
today = new Date()
): Omit<Poll, 'seen_at'>[] {
const out: Omit<Poll, 'seen_at'>[] = [];
const tables = findTables(html)
.map((t) => ({
g: tableGrid(t.html),
year: Number(t.heading.match(/\b(19|20)\d{2}\b/)?.[0] ?? today.getFullYear())
}))
.filter(({ g }) => {
const head = (g[0] ?? []).map((c) => c.text);
return s.match.every((m) => head.some((h) => h.startsWith(m)));
})
.slice(0, s.tables);
const limit = new Date(today.getTime() + 7 * 864e5).toISOString().slice(0, 10);
for (const { g, year } of tables) {
const head = g[0].map((c) => c.text.replace(/\s*\/\s*/g, '/'));
const col = (re: RegExp) => head.findIndex((h) => re.test(h));
const [cPoll, cDate, cSample] = [col(s.pollster), col(s.date), s.sample ? col(s.sample) : -1];
const options = head
.map((h, i) => ({ h, i }))
.filter(
({ h, i }) =>
i !== cPoll &&
i !== cDate &&
i !== cSample &&
h &&
!s.skip.test(h) &&
head.indexOf(h) === i
);
for (const row of g.slice(1)) {
if (row.every((c) => c.header) || row.length < head.length - 2) continue;
const dateCell: Cell | undefined = row[cDate];
if (!dateCell) continue;
let end = fieldworkEnd(dateCell.text, dateCell.sort, year);
if (end && end > limit) end = `${Number(end.slice(0, 4)) - 1}${end.slice(4)}`;
if (!end) continue;
const pollster = row[cPoll]?.text ?? '';
if (!pollster || pollster === dateCell.text) continue;
const sample = cSample >= 0 ? firstNumber(row[cSample]?.text ?? '') : null;
const key = `${end}|${pollster}|${sample ?? ''}`;
for (const { h, i } of options) {
const v = firstNumber(row[i]?.text ?? '');
if (v != null && !row[i]?.header)
out.push({
series: s.series,
poll_key: key,
fieldwork_end: end,
pollster,
sample,
option: h,
value: v
});
}
}
}
// Same poll twice (repeated across tables): keep the first.
const seen = new Set<string>();
return out.filter((p) => {
const k = `${p.poll_key}|${p.option}`;
if (seen.has(k)) return false;
seen.add(k);
return true;
});
}
export async function wikipolls(opts: { job: string }) {
await run('wikipedia', opts.job, async (note) => {
const pages = new Map<string, { revid: number; html: string; wikitext: string }>();
let rows = 0;
for (const s of WIKI_SERIES) {
let page = pages.get(s.page);
if (!page) {
const url = `${API}?action=parse&page=${encodeURIComponent(s.page)}&prop=text|wikitext|revid&redirects=1&format=json&formatversion=2`;
const res = await getJson<{
parse?: { revid: number; text: string; wikitext: string };
error?: { info: string };
}>(url);
if (!res.parse) {
note(`${s.page}: ${res.error?.info ?? 'no parse result'}`);
continue;
}
page = { revid: res.parse.revid, html: res.parse.text, wikitext: res.parse.wikitext };
pages.set(s.page, page);
await sql`
insert into stats.op_wiki_snapshot (page, revid, fetched_at, wikitext)
values (${s.page}, ${page.revid}, now(), ${page.wikitext})
on conflict (page, revid) do nothing`;
}
const polls = pollsFromHtml(s, page.html);
if (!polls.length) {
note(`${s.series}: no polls parsed from ${s.page} rev ${page.revid}`);
continue;
}
const now = new Date();
await sql.begin(async (tx) => {
await insertChunks(
tx,
'op_poll',
polls.map((p) => ({ ...p, seen_at: now })),
'on conflict (series, poll_key, option) do update set value = excluded.value, seen_at = excluded.seen_at'
);
});
log(`wikipolls ${s.series}: ${polls.length} values`);
rows += polls.length;
}
return rows;
});
}

1677
scripts/stats/world-opinions-seed.json Normal file
View File

File diff suppressed because it is too large Load Diff