stats elections: CBS 37278 percentages to votes; Kiesraad municipal votes and turnout from EML kieskring counts; ballots include blank/invalid

scripts/stats/elections.ts

@@ -1,5 +1,5 @@
 // A14 elections: NL Tweede Kamer from the Kiesraad open data (data.overheid.nl, CC0) and CBS
-// 37278 (1918–2006), US presidential results by state from the MIT Election Lab (CC0).
+// 37278 (1918–2006; votes there are derived from CBS's one-decimal percentages), US presidential results by state from the MIT Election Lab (CC0).
 //  - Kiesraad: per election the "CSV formaat" zip (votes per list per municipality and per
 //    polling station; 2023+ also turnout and seats) plus the EML zips, read with HTTP ranges, for
 //    the official result (seats, election date) and, before 2023, the national total count.
@@ -123,6 +123,22 @@ function parseTotaaltelling(xml: string) {
 	};
 }
 
+/** EML 510c (kieskring count): votes and turnout per municipality (ReportingUnit "HSBn::0513"). */
+function parseKieskringUnits(xml: string) {
+	const out: { code: string; name: string; t: ReturnType<typeof parseTotaaltelling> }[] = [];
+	for (const m of xml.matchAll(/<ReportingUnitVotes>([\s\S]*?)<\/ReportingUnitVotes>/g)) {
+		const id = m[1].match(/<ReportingUnitIdentifier Id="[^"]*::(\d{4})">([^<]*)</);
+		if (!id) continue;
+		const name = xmlText(id[2].replace(/^Gemeente\s+/i, '').trim());
+		out.push({
+			code: `GM${id[1]}`,
+			name,
+			t: parseTotaaltelling(`<TotalVotes>${m[1]}</TotalVotes>`)
+		});
+	}
+	return out;
+}
+
 type CkanResource = { url: string; name?: string; format?: string };
 type CkanPackage = { name: string; title: string; resources: CkanResource[] };
 
@@ -150,6 +166,8 @@ async function kiesraad(force: boolean, note: (w: string) => void) {
 		// EML zips: find the official result (and, for older years, the national count).
 		let resultaat: ReturnType<typeof parseResultaat> | null = null;
 		let totaal: ReturnType<typeof parseTotaaltelling> | null = null;
+		const hasUitslag = csvZip.find(/_uitslag\.csv$/i).length > 0;
+		const units = new Map<string, ReturnType<typeof parseKieskringUnits>[number]>();
 		for (const r of p.resources.filter((r) => r !== csvRes && /\.zip$/i.test(r.url))) {
 			const zip = await RemoteZip.open(r.url);
 			const re = zip.find(/(^|\/)Resultaat_TK\d+[^/]*\.eml\.xml$/i)[0];
@@ -157,7 +175,13 @@ async function kiesraad(force: boolean, note: (w: string) => void) {
 			const tt = zip.find(/(^|\/)Totaaltelling_TK\d+[^/]*\.eml\.xml$/i)[0];
 			if (tt && !totaal && !csvZip.find(/_uitslag\.csv$/i).length)
 				totaal = parseTotaaltelling(await zip.text(tt));
-			if (resultaat && (totaal || csvZip.find(/_uitslag\.csv$/i).length)) break;
+			// Before 2023 the CSV zip has no municipal turnout (and 2017's municipal CSV misses
+			// ~110 municipalities): take both from the kieskring counts where a zip has them.
+			if (!hasUitslag)
+				for (const k of zip.find(/Telling_TK\d+_kieskring_[^/]*\.eml\.xml$/i))
+					for (const u of parseKieskringUnits(await zip.text(k)))
+						if (!units.has(u.code)) units.set(u.code, u);
+			if (resultaat && (totaal || hasUitslag) && (hasUitslag || units.size)) break;
 		}
 		const results: Result[] = [];
 		const turnout: Turnout[] = [];
@@ -227,17 +251,43 @@ async function kiesraad(force: boolean, note: (w: string) => void) {
 			}
 		} else {
 			const gem = csvZip.find(/Per_Gemeente\.csv$/i)[0];
-			if (!gem) throw new Error(`${id}: no municipality CSV in ${csvRes.url}`);
-			for (const r of rowsOf(await csvZip.text(gem)))
-				results.push({
+			if (!gem && !units.size) throw new Error(`${id}: no municipality CSV in ${csvRes.url}`);
+			for (const u of units.values()) {
+				for (const [party, votes] of u.t.votes)
+					results.push({
+						...base,
+						level: 'muni',
+						area_code: u.code,
+						area_name: u.name,
+						party,
+						votes,
+						seats: null
+					});
+				const valid = [...u.t.votes.values()].reduce((a, b) => a + b, 0);
+				turnout.push({
 					...base,
 					level: 'muni',
-					area_code: gm(r.GemeenteCode),
-					area_name: r.GemeenteNaam,
-					party: r.PartijNaam,
-					votes: num(r.AantalStemmen),
-					seats: null
+					area_code: u.code,
+					area_name: u.name,
+					electorate: u.t.electorate,
+					ballots: valid + (u.t.blank ?? 0) + (u.t.invalid ?? 0),
+					valid,
+					blank: u.t.blank,
+					invalid: u.t.invalid
 				});
+			}
+			if (units.size) note(`${id}: ${units.size} municipalities from EML kieskring counts`);
+			for (const r of gem ? rowsOf(await csvZip.text(gem)) : [])
+				if (!units.has(gm(r.GemeenteCode)))
+					results.push({
+						...base,
+						level: 'muni',
+						area_code: gm(r.GemeenteCode),
+						area_name: r.GemeenteNaam,
+						party: r.PartijNaam,
+						votes: num(r.AantalStemmen),
+						seats: null
+					});
 			if (totaal) {
 				const seats = resultaat?.seats ?? new Map<string, number>();
 				for (const [party, votes] of totaal.votes)
@@ -257,7 +307,8 @@ async function kiesraad(force: boolean, note: (w: string) => void) {
 					area_code: 'NL',
 					area_name: 'Nederland',
 					electorate: totaal.electorate,
-					ballots: totaal.ballots,
+					// <TotalCounted> is the valid count in these files; ballots include blank and invalid.
+					ballots: valid + (totaal.blank ?? 0) + (totaal.invalid ?? 0),
 					valid,
 					blank: totaal.blank,
 					invalid: totaal.invalid
@@ -410,6 +461,14 @@ async function cbs37278() {
 				results.set(p.Title, r);
 			}
 		}
+		// 37278 gives vote shares per party (%, 1 decimal) and turnout as a %: convert to
+		// (rounded) vote counts against the valid-vote total so all elections share one shape.
+		const opkomst = d.Opkomst_2 == null ? null : Number(d.Opkomst_2);
+		if (turnout.ballots == null && opkomst != null && turnout.electorate != null)
+			turnout.ballots = Math.round((opkomst / 100) * turnout.electorate);
+		for (const r of results.values())
+			if (r.votes != null)
+				r.votes = turnout.valid != null ? Math.round((r.votes / 100) * turnout.valid) : null;
 		const list = [...results.values()].filter((r) => r.votes || r.seats);
 		const totalSeats = list.reduce((a, r) => a + (r.seats ?? 0), 0) || null;
 		rows += await save(