Stan Runge
e1fcd4f321
All checks were successful
Deploy production / deploy (push) Successful in 1m14s
603 lines
20 KiB
TypeScript
603 lines
20 KiB
TypeScript
// A14 elections: NL Tweede Kamer from the Kiesraad open data (data.overheid.nl, CC0) and CBS
|
||
// 37278 (1918–2006; votes there are derived from CBS's one-decimal percentages), US presidential results by state from the MIT Election Lab (CC0).
|
||
// - Kiesraad: per election the "CSV formaat" zip (votes per list per municipality and per
|
||
// polling station; 2023+ also turnout and seats) plus the EML zips, read with HTTP ranges, for
|
||
// the official result (seats, election date) and, before 2023, the national total count.
|
||
// Polling stations are kept for Gouda (GM0513) only.
|
||
// - Results are final, so an election already loaded is skipped unless --force.
|
||
import { insertChunks, getJson, http, log, run, sql } from './common';
|
||
import { parseCsv } from './owid';
|
||
import { RemoteZip } from './remotezip';
|
||
|
||
const CKAN =
|
||
'https://data.overheid.nl/data/api/3/action/package_search?q=verkiezingsuitslag%20Tweede%20Kamer&rows=100';
|
||
const STATION_MUNIS = new Set(['GM0513']);
|
||
|
||
type Result = {
|
||
election_id: string;
|
||
level: string;
|
||
area_code: string;
|
||
area_name: string | null;
|
||
party: string;
|
||
votes: number | null;
|
||
seats: number | null;
|
||
};
|
||
type Turnout = {
|
||
election_id: string;
|
||
level: string;
|
||
area_code: string;
|
||
area_name: string | null;
|
||
electorate: number | null;
|
||
ballots: number | null;
|
||
valid: number | null;
|
||
blank: number | null;
|
||
invalid: number | null;
|
||
};
|
||
type Election = {
|
||
election_id: string;
|
||
country: string;
|
||
kind: string;
|
||
date: string;
|
||
name: string;
|
||
source_id: string;
|
||
total_seats: number | null;
|
||
};
|
||
|
||
async function save(e: Election, results: Result[], turnout: Turnout[]) {
|
||
// Merge duplicate keys (a list name can repeat within a municipality in old files).
|
||
const merged = new Map<string, Result>();
|
||
for (const r of results) {
|
||
const key = `${r.level}|${r.area_code}|${r.party}`;
|
||
const prev = merged.get(key);
|
||
if (!prev) merged.set(key, { ...r });
|
||
else {
|
||
prev.votes =
|
||
prev.votes == null && r.votes == null ? null : (prev.votes ?? 0) + (r.votes ?? 0);
|
||
prev.seats = prev.seats ?? r.seats;
|
||
}
|
||
}
|
||
await sql.begin(async (tx) => {
|
||
await tx`delete from stats.elec_result where election_id = ${e.election_id}`;
|
||
await tx`delete from stats.elec_turnout where election_id = ${e.election_id}`;
|
||
await tx`delete from stats.elec_election where election_id = ${e.election_id}`;
|
||
await insertChunks(tx, 'elec_election', [e]);
|
||
await insertChunks(tx, 'elec_result', [...merged.values()]);
|
||
await insertChunks(tx, 'elec_turnout', turnout);
|
||
});
|
||
return merged.size + turnout.length;
|
||
}
|
||
|
||
const num = (s: string | undefined) => {
|
||
if (s == null || s.trim() === '') return null;
|
||
const n = Number(s.replace(/\s/g, ''));
|
||
return Number.isFinite(n) ? n : null;
|
||
};
|
||
const gm = (code: string) => `GM${code.replace(/^\D+/, '').padStart(4, '0')}`;
|
||
const xmlText = (s: string) =>
|
||
s
|
||
.replace(/</g, '<')
|
||
.replace(/>/g, '>')
|
||
.replace(/"/g, '"')
|
||
.replace(/'/g, "'")
|
||
.replace(/&/g, '&');
|
||
|
||
/** Rows of a semicolon CSV as objects keyed by the header. */
|
||
function rowsOf(text: string): Record<string, string>[] {
|
||
const [header, ...rows] = parseCsv(text, ';');
|
||
return rows
|
||
.filter((r) => r.length >= header.length - 1 && r.some((x) => x !== ''))
|
||
.map((r) => Object.fromEntries(header.map((h, i) => [h.trim(), (r[i] ?? '').trim()])));
|
||
}
|
||
|
||
/** EML 520 (Resultaat): election date and seats per list (elected candidates per affiliation). */
|
||
function parseResultaat(xml: string) {
|
||
const date = xml.match(/<kr:ElectionDate>([\d-]+)<\/kr:ElectionDate>/)?.[1] ?? null;
|
||
const seats = new Map<string, number>();
|
||
let current = '';
|
||
for (const m of xml.matchAll(/<Selection>([\s\S]*?)<\/Selection>/g)) {
|
||
const aff = m[1].match(/<RegisteredName>([\s\S]*?)<\/RegisteredName>/);
|
||
if (aff) {
|
||
current = xmlText(aff[1].trim());
|
||
if (!seats.has(current)) seats.set(current, 0);
|
||
} else if (/<Candidate>/.test(m[1]) && /<Elected>yes<\/Elected>/.test(m[1]))
|
||
seats.set(current, (seats.get(current) ?? 0) + 1);
|
||
}
|
||
return { date, seats };
|
||
}
|
||
|
||
/** EML 510d (Totaaltelling): national votes per list and the count totals. */
|
||
function parseTotaaltelling(xml: string) {
|
||
const total = xml.match(/<TotalVotes>([\s\S]*?)<\/TotalVotes>/)?.[1] ?? '';
|
||
const votes = new Map<string, number>();
|
||
for (const m of total.matchAll(
|
||
/<Selection><AffiliationIdentifier[^>]*><RegisteredName>([\s\S]*?)<\/RegisteredName><\/AffiliationIdentifier><ValidVotes>(\d+)<\/ValidVotes>/g
|
||
))
|
||
votes.set(xmlText(m[1].trim()), Number(m[2]));
|
||
const pick = (re: RegExp) => num(total.match(re)?.[1]);
|
||
return {
|
||
votes,
|
||
electorate: pick(/<Cast>(\d+)<\/Cast>/),
|
||
ballots: pick(/<TotalCounted>(\d+)<\/TotalCounted>/),
|
||
invalid: pick(/<RejectedVotes ReasonCode="ongeldig">(\d+)</),
|
||
blank: pick(/<RejectedVotes ReasonCode="blanco">(\d+)</)
|
||
};
|
||
}
|
||
|
||
/** EML 510c (kieskring count): votes and turnout per municipality (ReportingUnit "HSBn::0513"). */
|
||
function parseKieskringUnits(xml: string) {
|
||
const out: { code: string; name: string; t: ReturnType<typeof parseTotaaltelling> }[] = [];
|
||
for (const m of xml.matchAll(/<ReportingUnitVotes>([\s\S]*?)<\/ReportingUnitVotes>/g)) {
|
||
const id = m[1].match(/<ReportingUnitIdentifier Id="[^"]*::(\d{4})">([^<]*)</);
|
||
if (!id) continue;
|
||
const name = xmlText(id[2].replace(/^Gemeente\s+/i, '').trim());
|
||
out.push({
|
||
code: `GM${id[1]}`,
|
||
name,
|
||
t: parseTotaaltelling(`<TotalVotes>${m[1]}</TotalVotes>`)
|
||
});
|
||
}
|
||
return out;
|
||
}
|
||
|
||
type CkanResource = { url: string; name?: string; format?: string };
|
||
type CkanPackage = { name: string; title: string; resources: CkanResource[] };
|
||
|
||
async function kiesraad(force: boolean, note: (w: string) => void) {
|
||
const res = await getJson<{ result: { results: CkanPackage[] } }>(CKAN);
|
||
const packages = res.result.results
|
||
.map((p) => ({ p, year: p.name.match(/^verkiezingsuitslag-tweede-kamer-(\d{4})$/)?.[1] }))
|
||
.filter((x): x is { p: CkanPackage; year: string } => !!x.year)
|
||
.sort((a, b) => a.year.localeCompare(b.year));
|
||
const loaded = new Set(
|
||
(await sql`select election_id from stats.elec_election where source_id = 'kiesraad'`).map(
|
||
(r) => r.election_id as string
|
||
)
|
||
);
|
||
let rows = 0;
|
||
for (const { p, year } of packages) {
|
||
const id = `NL-TK-${year}`;
|
||
if (loaded.has(id) && !force) continue;
|
||
const csvRes = p.resources.find((r) => /csv/i.test(decodeURIComponent(r.url)));
|
||
if (!csvRes) {
|
||
note(`${id}: no CSV zip`);
|
||
continue;
|
||
}
|
||
const csvZip = await RemoteZip.open(csvRes.url);
|
||
// EML zips: find the official result (and, for older years, the national count).
|
||
let resultaat: ReturnType<typeof parseResultaat> | null = null;
|
||
let totaal: ReturnType<typeof parseTotaaltelling> | null = null;
|
||
const hasUitslag = csvZip.find(/_uitslag\.csv$/i).length > 0;
|
||
const units = new Map<string, ReturnType<typeof parseKieskringUnits>[number]>();
|
||
for (const r of p.resources.filter((r) => r !== csvRes && /\.zip$/i.test(r.url))) {
|
||
const zip = await RemoteZip.open(r.url);
|
||
const re = zip.find(/(^|\/)Resultaat_TK\d+[^/]*\.eml\.xml$/i)[0];
|
||
if (re && !resultaat) resultaat = parseResultaat(await zip.text(re));
|
||
const tt = zip.find(/(^|\/)Totaaltelling_TK\d+[^/]*\.eml\.xml$/i)[0];
|
||
if (tt && !totaal && !csvZip.find(/_uitslag\.csv$/i).length)
|
||
totaal = parseTotaaltelling(await zip.text(tt));
|
||
// Before 2023 the CSV zip has no municipal turnout (and 2017's municipal CSV misses
|
||
// ~110 municipalities): take both from the kieskring counts where a zip has them.
|
||
if (!hasUitslag)
|
||
for (const k of zip.find(/Telling_TK\d+_kieskring_[^/]*\.eml\.xml$/i))
|
||
for (const u of parseKieskringUnits(await zip.text(k)))
|
||
if (!units.has(u.code)) units.set(u.code, u);
|
||
if (resultaat && (totaal || hasUitslag) && (hasUitslag || units.size)) break;
|
||
}
|
||
const results: Result[] = [];
|
||
const turnout: Turnout[] = [];
|
||
const base = { election_id: id };
|
||
const uitslag = csvZip.find(/_uitslag\.csv$/i)[0];
|
||
if (uitslag) {
|
||
for (const r of rowsOf(await csvZip.text(uitslag))) {
|
||
const code = r.RegioCode ?? '';
|
||
const level = code.startsWith('L') ? 'national' : /^[GO]/.test(code) ? 'muni' : null;
|
||
if (!level) continue;
|
||
const area_code = level === 'national' ? 'NL' : gm(code);
|
||
const area_name = level === 'national' ? 'Nederland' : r.Regio;
|
||
const v = num(r.Waarde);
|
||
if (r.VeldType === 'LijstAantalStemmen' && r.LijstNaam)
|
||
results.push({
|
||
...base,
|
||
level,
|
||
area_code,
|
||
area_name,
|
||
party: r.LijstNaam,
|
||
votes: v,
|
||
seats: null
|
||
});
|
||
else if (r.VeldType === 'LijstAantalZetels' && r.LijstNaam && level === 'national')
|
||
results.push({
|
||
...base,
|
||
level,
|
||
area_code,
|
||
area_name,
|
||
party: r.LijstNaam,
|
||
votes: null,
|
||
seats: v
|
||
});
|
||
else if (
|
||
[
|
||
'Kiesgerechtigden',
|
||
'Opkomst',
|
||
'AantalGeldigeStemmen',
|
||
'AantalBlancoStemmen',
|
||
'AantalOngeldigeStemmen'
|
||
].includes(r.VeldType)
|
||
) {
|
||
let t = turnout.find((x) => x.level === level && x.area_code === area_code);
|
||
if (!t) {
|
||
t = {
|
||
...base,
|
||
level,
|
||
area_code,
|
||
area_name,
|
||
electorate: null,
|
||
ballots: null,
|
||
valid: null,
|
||
blank: null,
|
||
invalid: null
|
||
};
|
||
turnout.push(t);
|
||
}
|
||
const col = {
|
||
Kiesgerechtigden: 'electorate',
|
||
Opkomst: 'ballots',
|
||
AantalGeldigeStemmen: 'valid',
|
||
AantalBlancoStemmen: 'blank',
|
||
AantalOngeldigeStemmen: 'invalid'
|
||
}[r.VeldType] as 'electorate' | 'ballots' | 'valid' | 'blank' | 'invalid';
|
||
t[col] = v;
|
||
}
|
||
}
|
||
} else {
|
||
const gem = csvZip.find(/Per_Gemeente\.csv$/i)[0];
|
||
if (!gem && !units.size) throw new Error(`${id}: no municipality CSV in ${csvRes.url}`);
|
||
for (const u of units.values()) {
|
||
for (const [party, votes] of u.t.votes)
|
||
results.push({
|
||
...base,
|
||
level: 'muni',
|
||
area_code: u.code,
|
||
area_name: u.name,
|
||
party,
|
||
votes,
|
||
seats: null
|
||
});
|
||
const valid = [...u.t.votes.values()].reduce((a, b) => a + b, 0);
|
||
turnout.push({
|
||
...base,
|
||
level: 'muni',
|
||
area_code: u.code,
|
||
area_name: u.name,
|
||
electorate: u.t.electorate,
|
||
ballots: valid + (u.t.blank ?? 0) + (u.t.invalid ?? 0),
|
||
valid,
|
||
blank: u.t.blank,
|
||
invalid: u.t.invalid
|
||
});
|
||
}
|
||
if (units.size) note(`${id}: ${units.size} municipalities from EML kieskring counts`);
|
||
for (const r of gem ? rowsOf(await csvZip.text(gem)) : [])
|
||
if (!units.has(gm(r.GemeenteCode)))
|
||
results.push({
|
||
...base,
|
||
level: 'muni',
|
||
area_code: gm(r.GemeenteCode),
|
||
area_name: r.GemeenteNaam,
|
||
party: r.PartijNaam,
|
||
votes: num(r.AantalStemmen),
|
||
seats: null
|
||
});
|
||
if (totaal) {
|
||
const seats = resultaat?.seats ?? new Map<string, number>();
|
||
for (const [party, votes] of totaal.votes)
|
||
results.push({
|
||
...base,
|
||
level: 'national',
|
||
area_code: 'NL',
|
||
area_name: 'Nederland',
|
||
party,
|
||
votes,
|
||
seats: seats.get(party) ?? null
|
||
});
|
||
const valid = [...totaal.votes.values()].reduce((a, b) => a + b, 0);
|
||
turnout.push({
|
||
...base,
|
||
level: 'national',
|
||
area_code: 'NL',
|
||
area_name: 'Nederland',
|
||
electorate: totaal.electorate,
|
||
// <TotalCounted> is the valid count in these files; ballots include blank and invalid.
|
||
ballots: valid + (totaal.blank ?? 0) + (totaal.invalid ?? 0),
|
||
valid,
|
||
blank: totaal.blank,
|
||
invalid: totaal.invalid
|
||
});
|
||
} else note(`${id}: no Totaaltelling EML, national total missing`);
|
||
}
|
||
// Merge votes and seats rows of the national level (2023+ lists them separately).
|
||
const nat = new Map<string, Result>();
|
||
for (const r of results.filter((r) => r.level === 'national')) {
|
||
const prev = nat.get(r.party);
|
||
if (prev) {
|
||
prev.votes ??= r.votes;
|
||
prev.seats ??= r.seats;
|
||
} else nat.set(r.party, r);
|
||
}
|
||
const finalResults = [...results.filter((r) => r.level !== 'national'), ...nat.values()];
|
||
// Polling stations (Gouda only).
|
||
const sb = csvZip.find(/Per_Stembureau\.csv$/i)[0];
|
||
if (sb) {
|
||
for (const r of rowsOf(await csvZip.text(sb))) {
|
||
const muni = gm(r.GemeenteCode);
|
||
if (!STATION_MUNIS.has(muni)) continue;
|
||
finalResults.push({
|
||
...base,
|
||
level: 'station',
|
||
area_code: `${muni}-${r.StembureauCode}`,
|
||
area_name: r.StembureauNaam || null,
|
||
party: r.PartijNaam,
|
||
votes: num(r.AantalStemmen),
|
||
seats: null
|
||
});
|
||
}
|
||
} else note(`${id}: no polling-station CSV`);
|
||
const totalSeats = [...nat.values()].reduce((a, r) => a + (r.seats ?? 0), 0) || null;
|
||
const date = resultaat?.date ?? KNOWN_DATES[year];
|
||
if (!date) throw new Error(`${id}: no election date`);
|
||
if (!resultaat) note(`${id}: no Resultaat EML (seats/date from CSV or table)`);
|
||
const n = await save(
|
||
{
|
||
election_id: id,
|
||
country: 'NLD',
|
||
kind: 'TK',
|
||
date,
|
||
name: `Tweede Kamer ${year}`,
|
||
source_id: 'kiesraad',
|
||
total_seats: totalSeats
|
||
},
|
||
finalResults,
|
||
turnout
|
||
);
|
||
log(
|
||
`kiesraad ${id}: ${n} rows (${finalResults.filter((r) => r.level === 'station').length} Gouda station rows)`
|
||
);
|
||
rows += n;
|
||
}
|
||
return rows;
|
||
}
|
||
|
||
// Election days (CBS 37278 only has the year). Source: Kiesraad / Parlement.com.
|
||
const KNOWN_DATES: Record<string, string> = {
|
||
'1918': '1918-07-03',
|
||
'1922': '1922-07-05',
|
||
'1925': '1925-07-01',
|
||
'1929': '1929-07-03',
|
||
'1933': '1933-04-26',
|
||
'1937': '1937-05-26',
|
||
'1946': '1946-05-17',
|
||
'1948': '1948-07-07',
|
||
'1952': '1952-06-25',
|
||
'1956': '1956-06-13',
|
||
'1959': '1959-03-12',
|
||
'1963': '1963-05-15',
|
||
'1967': '1967-02-15',
|
||
'1971': '1971-04-28',
|
||
'1972': '1972-11-29',
|
||
'1977': '1977-05-25',
|
||
'1981': '1981-05-26',
|
||
'1982': '1982-09-08',
|
||
'1986': '1986-05-21',
|
||
'1989': '1989-09-06',
|
||
'1994': '1994-05-03',
|
||
'1998': '1998-05-06',
|
||
'2002': '2002-05-15',
|
||
'2003': '2003-01-22',
|
||
'2006': '2006-11-22',
|
||
'2010': '2010-06-09',
|
||
'2012': '2012-09-12',
|
||
'2017': '2017-03-15',
|
||
'2021': '2021-03-17',
|
||
'2023': '2023-11-22',
|
||
'2025': '2025-10-29'
|
||
};
|
||
|
||
type CbsProp = { Key: string; Title: string; Type: string; ParentID?: number | null; ID: number };
|
||
|
||
async function cbs37278() {
|
||
const base = 'https://opendata.cbs.nl/ODataApi/odata/37278';
|
||
const props = (await getJson<{ value: CbsProp[] }>(`${base}/DataProperties`)).value;
|
||
const groups = new Map(props.filter((p) => p.Type === 'TopicGroup').map((p) => [p.ID, p.Title]));
|
||
const data = (
|
||
await getJson<{ value: Record<string, string | number | null>[] }>(`${base}/TypedDataSet`)
|
||
).value;
|
||
let rows = 0;
|
||
for (const d of data) {
|
||
const year = String(d.Perioden).slice(0, 4);
|
||
const id = `NL-TK-${year}`;
|
||
const results = new Map<string, Result>();
|
||
const turnout: Turnout = {
|
||
election_id: id,
|
||
level: 'national',
|
||
area_code: 'NL',
|
||
area_name: 'Nederland',
|
||
electorate: null,
|
||
ballots: null,
|
||
valid: null,
|
||
blank: null,
|
||
invalid: null
|
||
};
|
||
for (const p of props.filter((p) => p.Type === 'Topic')) {
|
||
const v = d[p.Key] == null ? null : Number(d[p.Key]);
|
||
const group = groups.get(p.ParentID ?? -1) ?? '';
|
||
if (p.Key.startsWith('Kiesgerechtigden')) turnout.electorate = v;
|
||
else if (p.Key.startsWith('TotaalStemmen')) turnout.ballots = v;
|
||
else if (p.Key.startsWith('OngeldigeStemmen')) turnout.invalid = v;
|
||
else if (p.Key.startsWith('TotaalGeldigeStemmen')) turnout.valid = v;
|
||
else if (/^(Opkomst|TotaalAantalZetels)/.test(p.Key)) continue;
|
||
else if (v != null && /stemmen per partij/i.test(group)) {
|
||
const r = results.get(p.Title) ?? {
|
||
election_id: id,
|
||
level: 'national',
|
||
area_code: 'NL',
|
||
area_name: 'Nederland',
|
||
party: p.Title,
|
||
votes: null,
|
||
seats: null
|
||
};
|
||
r.votes = v;
|
||
results.set(p.Title, r);
|
||
} else if (v != null && /zetel/i.test(group)) {
|
||
const r = results.get(p.Title) ?? {
|
||
election_id: id,
|
||
level: 'national',
|
||
area_code: 'NL',
|
||
area_name: 'Nederland',
|
||
party: p.Title,
|
||
votes: null,
|
||
seats: null
|
||
};
|
||
r.seats = v;
|
||
results.set(p.Title, r);
|
||
}
|
||
}
|
||
// 37278 gives vote shares per party (%, 1 decimal) and turnout as a %: convert to
|
||
// (rounded) vote counts against the valid-vote total so all elections share one shape.
|
||
const opkomst = d.Opkomst_2 == null ? null : Number(d.Opkomst_2);
|
||
if (turnout.ballots == null && opkomst != null && turnout.electorate != null)
|
||
turnout.ballots = Math.round((opkomst / 100) * turnout.electorate);
|
||
for (const r of results.values())
|
||
if (r.votes != null)
|
||
r.votes = turnout.valid != null ? Math.round((r.votes / 100) * turnout.valid) : null;
|
||
const list = [...results.values()].filter((r) => r.votes || r.seats);
|
||
const totalSeats = list.reduce((a, r) => a + (r.seats ?? 0), 0) || null;
|
||
rows += await save(
|
||
{
|
||
election_id: id,
|
||
country: 'NLD',
|
||
kind: 'TK',
|
||
date: KNOWN_DATES[year] ?? `${year}-01-01`,
|
||
name: `Tweede Kamer ${year}`,
|
||
source_id: 'cbs',
|
||
total_seats: totalSeats
|
||
},
|
||
list,
|
||
[turnout]
|
||
);
|
||
}
|
||
return rows;
|
||
}
|
||
|
||
/** US presidential election day: the Tuesday after the first Monday in November. */
|
||
function usElectionDay(year: number) {
|
||
const d = new Date(Date.UTC(year, 10, 1));
|
||
const firstMonday = 1 + ((8 - d.getUTCDay()) % 7);
|
||
return `${year}-11-${String(firstMonday + 1).padStart(2, '0')}`;
|
||
}
|
||
|
||
async function mit(note: (w: string) => void) {
|
||
const ds = await getJson<{
|
||
data: { latestVersion: { files: { dataFile: { id: number; filename: string } }[] } };
|
||
}>(
|
||
'https://dataverse.harvard.edu/api/datasets/:persistentId/?persistentId=doi:10.7910/DVN/42MVDX'
|
||
);
|
||
const file = ds.data.latestVersion.files
|
||
.map((f) => f.dataFile)
|
||
.filter((f) => /^\d{4}-\d{4}-president\.(csv|tab)$/.test(f.filename))
|
||
.sort((a, b) => b.filename.localeCompare(a.filename))[0];
|
||
if (!file) throw new Error('MIT president file not found in doi:10.7910/DVN/42MVDX');
|
||
const text = await (
|
||
await http(`https://dataverse.harvard.edu/api/access/datafile/${file.id}`)
|
||
).text();
|
||
const sep = file.filename.endsWith('.tab') ? '\t' : ',';
|
||
const [header, ...lines] = parseCsv(text, sep);
|
||
const col = (name: string) => header.indexOf(name);
|
||
const [cYear, cState, cPo, cParty, cVotes, cTotal] = [
|
||
'year',
|
||
'state',
|
||
'state_po',
|
||
'party_simplified',
|
||
'candidatevotes',
|
||
'totalvotes'
|
||
].map(col);
|
||
if ([cYear, cState, cPo, cParty, cVotes, cTotal].some((i) => i < 0))
|
||
throw new Error(`MIT columns changed: ${header.join(',')}`);
|
||
const byYear = new Map<number, { results: Map<string, Result>; turnout: Map<string, Turnout> }>();
|
||
for (const l of lines) {
|
||
if (l.length < header.length - 1) continue;
|
||
const year = Number(l[cYear]);
|
||
const id = `US-PRES-${year}`;
|
||
const y = byYear.get(year) ?? { results: new Map(), turnout: new Map() };
|
||
byYear.set(year, y);
|
||
const state = l[cPo];
|
||
const party = l[cParty] || 'OTHER';
|
||
const votes = Number(l[cVotes]) || 0;
|
||
for (const [level, area_code, area_name] of [
|
||
['state', state, l[cState]],
|
||
['national', 'US', 'United States']
|
||
]) {
|
||
const key = `${level}|${area_code}|${party}`;
|
||
const r = y.results.get(key) ?? {
|
||
election_id: id,
|
||
level,
|
||
area_code,
|
||
area_name,
|
||
party,
|
||
votes: 0,
|
||
seats: null
|
||
};
|
||
r.votes = (r.votes ?? 0) + votes;
|
||
y.results.set(key, r);
|
||
}
|
||
if (!y.turnout.has(state))
|
||
y.turnout.set(state, {
|
||
election_id: id,
|
||
level: 'state',
|
||
area_code: state,
|
||
area_name: l[cState],
|
||
electorate: null,
|
||
ballots: Number(l[cTotal]) || null,
|
||
valid: null,
|
||
blank: null,
|
||
invalid: null
|
||
});
|
||
}
|
||
let rows = 0;
|
||
for (const [year, y] of byYear) {
|
||
const states = [...y.turnout.values()];
|
||
states.push({
|
||
election_id: `US-PRES-${year}`,
|
||
level: 'national',
|
||
area_code: 'US',
|
||
area_name: 'United States',
|
||
electorate: null,
|
||
ballots: states.reduce((a, s) => a + (s.ballots ?? 0), 0),
|
||
valid: null,
|
||
blank: null,
|
||
invalid: null
|
||
});
|
||
rows += await save(
|
||
{
|
||
election_id: `US-PRES-${year}`,
|
||
country: 'USA',
|
||
kind: 'PRES',
|
||
date: usElectionDay(year),
|
||
name: `US presidential ${year}`,
|
||
source_id: 'mitelection',
|
||
total_seats: null
|
||
},
|
||
[...y.results.values()],
|
||
states
|
||
);
|
||
}
|
||
if (!byYear.size) note('MIT file had no rows');
|
||
return rows;
|
||
}
|
||
|
||
export async function elections(opts: { force: boolean; only?: string; job: string }) {
|
||
const want = (k: string) => !opts.only || opts.only.split(',').includes(k);
|
||
if (want('kiesraad')) await run('kiesraad', opts.job, (note) => kiesraad(opts.force, note));
|
||
if (want('cbs')) await run('cbs', `${opts.job}:37278`, () => cbs37278());
|
||
if (want('mit')) await run('mitelection', opts.job, (note) => mit(note));
|
||
}
|