diff --git a/scripts/stats/elections.ts b/scripts/stats/elections.ts index 4c7c948..75b189a 100644 --- a/scripts/stats/elections.ts +++ b/scripts/stats/elections.ts @@ -1,5 +1,5 @@ // A14 elections: NL Tweede Kamer from the Kiesraad open data (data.overheid.nl, CC0) and CBS -// 37278 (1918–2006), US presidential results by state from the MIT Election Lab (CC0). +// 37278 (1918–2006; votes there are derived from CBS's one-decimal percentages), US presidential results by state from the MIT Election Lab (CC0). // - Kiesraad: per election the "CSV formaat" zip (votes per list per municipality and per // polling station; 2023+ also turnout and seats) plus the EML zips, read with HTTP ranges, for // the official result (seats, election date) and, before 2023, the national total count. @@ -123,6 +123,22 @@ function parseTotaaltelling(xml: string) { }; } +/** EML 510c (kieskring count): votes and turnout per municipality (ReportingUnit "HSBn::0513"). */ +function parseKieskringUnits(xml: string) { + const out: { code: string; name: string; t: ReturnType }[] = []; + for (const m of xml.matchAll(/([\s\S]*?)<\/ReportingUnitVotes>/g)) { + const id = m[1].match(/([^<]*)${m[1]}`) + }); + } + return out; +} + type CkanResource = { url: string; name?: string; format?: string }; type CkanPackage = { name: string; title: string; resources: CkanResource[] }; @@ -150,6 +166,8 @@ async function kiesraad(force: boolean, note: (w: string) => void) { // EML zips: find the official result (and, for older years, the national count). let resultaat: ReturnType | null = null; let totaal: ReturnType | null = null; + const hasUitslag = csvZip.find(/_uitslag\.csv$/i).length > 0; + const units = new Map[number]>(); for (const r of p.resources.filter((r) => r !== csvRes && /\.zip$/i.test(r.url))) { const zip = await RemoteZip.open(r.url); const re = zip.find(/(^|\/)Resultaat_TK\d+[^/]*\.eml\.xml$/i)[0]; @@ -157,7 +175,13 @@ async function kiesraad(force: boolean, note: (w: string) => void) { const tt = zip.find(/(^|\/)Totaaltelling_TK\d+[^/]*\.eml\.xml$/i)[0]; if (tt && !totaal && !csvZip.find(/_uitslag\.csv$/i).length) totaal = parseTotaaltelling(await zip.text(tt)); - if (resultaat && (totaal || csvZip.find(/_uitslag\.csv$/i).length)) break; + // Before 2023 the CSV zip has no municipal turnout (and 2017's municipal CSV misses + // ~110 municipalities): take both from the kieskring counts where a zip has them. + if (!hasUitslag) + for (const k of zip.find(/Telling_TK\d+_kieskring_[^/]*\.eml\.xml$/i)) + for (const u of parseKieskringUnits(await zip.text(k))) + if (!units.has(u.code)) units.set(u.code, u); + if (resultaat && (totaal || hasUitslag) && (hasUitslag || units.size)) break; } const results: Result[] = []; const turnout: Turnout[] = []; @@ -227,17 +251,43 @@ async function kiesraad(force: boolean, note: (w: string) => void) { } } else { const gem = csvZip.find(/Per_Gemeente\.csv$/i)[0]; - if (!gem) throw new Error(`${id}: no municipality CSV in ${csvRes.url}`); - for (const r of rowsOf(await csvZip.text(gem))) - results.push({ + if (!gem && !units.size) throw new Error(`${id}: no municipality CSV in ${csvRes.url}`); + for (const u of units.values()) { + for (const [party, votes] of u.t.votes) + results.push({ + ...base, + level: 'muni', + area_code: u.code, + area_name: u.name, + party, + votes, + seats: null + }); + const valid = [...u.t.votes.values()].reduce((a, b) => a + b, 0); + turnout.push({ ...base, level: 'muni', - area_code: gm(r.GemeenteCode), - area_name: r.GemeenteNaam, - party: r.PartijNaam, - votes: num(r.AantalStemmen), - seats: null + area_code: u.code, + area_name: u.name, + electorate: u.t.electorate, + ballots: valid + (u.t.blank ?? 0) + (u.t.invalid ?? 0), + valid, + blank: u.t.blank, + invalid: u.t.invalid }); + } + if (units.size) note(`${id}: ${units.size} municipalities from EML kieskring counts`); + for (const r of gem ? rowsOf(await csvZip.text(gem)) : []) + if (!units.has(gm(r.GemeenteCode))) + results.push({ + ...base, + level: 'muni', + area_code: gm(r.GemeenteCode), + area_name: r.GemeenteNaam, + party: r.PartijNaam, + votes: num(r.AantalStemmen), + seats: null + }); if (totaal) { const seats = resultaat?.seats ?? new Map(); for (const [party, votes] of totaal.votes) @@ -257,7 +307,8 @@ async function kiesraad(force: boolean, note: (w: string) => void) { area_code: 'NL', area_name: 'Nederland', electorate: totaal.electorate, - ballots: totaal.ballots, + // is the valid count in these files; ballots include blank and invalid. + ballots: valid + (totaal.blank ?? 0) + (totaal.invalid ?? 0), valid, blank: totaal.blank, invalid: totaal.invalid @@ -410,6 +461,14 @@ async function cbs37278() { results.set(p.Title, r); } } + // 37278 gives vote shares per party (%, 1 decimal) and turnout as a %: convert to + // (rounded) vote counts against the valid-vote total so all elections share one shape. + const opkomst = d.Opkomst_2 == null ? null : Number(d.Opkomst_2); + if (turnout.ballots == null && opkomst != null && turnout.electorate != null) + turnout.ballots = Math.round((opkomst / 100) * turnout.electorate); + for (const r of results.values()) + if (r.votes != null) + r.votes = turnout.valid != null ? Math.round((r.votes / 100) * turnout.valid) : null; const list = [...results.values()].filter((r) => r.votes || r.seats); const totalSeats = list.reduce((a, r) => a + (r.seats ?? 0), 0) || null; rows += await save(