Stan Runge
42c9351ebd
All checks were successful
Deploy production / deploy (push) Successful in 1m12s
249 lines
7.5 KiB
TypeScript
249 lines
7.5 KiB
TypeScript
// A15 party positions: CHES trend file (party means 1999–2024) and V-Party v2 (V-Dem, via the
|
||
// vdemdata GitHub repo's RData), linked through the Party Facts crosswalk. Writes stats.wo_party
|
||
// (data-driven rows; the curated guide parties keep their own `wo:` rows) and
|
||
// stats.wo_party_position. Curated rows get a partyfacts_id when a match is unambiguous.
|
||
import { tmpdir } from 'node:os';
|
||
import { join } from 'node:path';
|
||
import { insertChunks, http, log, run, sql } from './common';
|
||
import { parseCsv } from './owid';
|
||
|
||
const CHES_URL =
|
||
'https://github.com/chesdata/chesdata.github.io/releases/download/ches-trend/1999-2024_CHES_dataset_meansV2.csv';
|
||
const VPARTY_URL = 'https://github.com/vdeminstitute/vdemdata/raw/master/data/vparty.RData';
|
||
const PARTYFACTS_URL = 'https://partyfacts.herokuapp.com/download/external-parties-csv/';
|
||
const PYTHON = process.env.STATS_PYTHON ?? `${process.env.HOME}/.local/share/stats-py/bin/python`;
|
||
|
||
const CHES_COUNTRY: Record<number, string> = {
|
||
1: 'BEL',
|
||
2: 'DNK',
|
||
3: 'DEU',
|
||
4: 'GRC',
|
||
5: 'ESP',
|
||
6: 'FRA',
|
||
7: 'IRL',
|
||
8: 'ITA',
|
||
10: 'NLD',
|
||
11: 'GBR',
|
||
12: 'PRT',
|
||
13: 'AUT',
|
||
14: 'FIN',
|
||
16: 'SWE',
|
||
20: 'BGR',
|
||
21: 'CZE',
|
||
22: 'EST',
|
||
23: 'HUN',
|
||
24: 'LVA',
|
||
25: 'LTU',
|
||
26: 'POL',
|
||
27: 'ROU',
|
||
28: 'SVK',
|
||
29: 'SVN',
|
||
31: 'HRV',
|
||
37: 'MLT',
|
||
38: 'LUX',
|
||
40: 'CYP',
|
||
9: 'NOR',
|
||
35: 'NOR',
|
||
36: 'CHE',
|
||
34: 'TUR',
|
||
45: 'ISL'
|
||
};
|
||
const CHES_FAMILY: Record<number, string> = {
|
||
1: 'radical right',
|
||
2: 'conservative',
|
||
3: 'liberal',
|
||
4: 'christian democratic',
|
||
5: 'socialist',
|
||
6: 'radical left',
|
||
7: 'green',
|
||
8: 'regionalist',
|
||
9: 'no family',
|
||
10: 'confessional',
|
||
11: 'agrarian/centre'
|
||
};
|
||
const CHES_DIMS = ['lrgen', 'lrecon', 'galtan', 'eu_position', 'immigrate_policy'];
|
||
const VPARTY_DIMS = [
|
||
'v2xpa_popul',
|
||
'v2xpa_antiplural',
|
||
'v2pariglef',
|
||
'v2paimmig',
|
||
'v2palgbt',
|
||
'v2parelig',
|
||
'v2pawelf',
|
||
'v2paculsup'
|
||
];
|
||
|
||
type Party = {
|
||
party_id: string;
|
||
iso3: string | null;
|
||
name: string | null;
|
||
abbrev: string | null;
|
||
family: string | null;
|
||
ches_id: number | null;
|
||
manifesto_id: number | null;
|
||
vparty_id: number | null;
|
||
partyfacts_id: number | null;
|
||
sort: null;
|
||
};
|
||
type Position = {
|
||
party_id: string;
|
||
year: number;
|
||
source: string;
|
||
dim: string;
|
||
value: number | null;
|
||
vote_share: number | null;
|
||
};
|
||
|
||
const table = (text: string) => {
|
||
const [header, ...rows] = parseCsv(text);
|
||
return rows
|
||
.filter((r) => r.length >= header.length - 1)
|
||
.map((r) => Object.fromEntries(header.map((h, i) => [h, r[i] ?? ''])));
|
||
};
|
||
const num = (s: string | undefined) =>
|
||
s == null || s === '' || s === 'NA' ? null : Number.isFinite(Number(s)) ? Number(s) : null;
|
||
|
||
export async function parties(opts: { job: string }) {
|
||
await run('ches', opts.job, async (note) => {
|
||
// Party Facts: dataset ids → partyfacts ids (and countries).
|
||
const pf = table(await (await http(PARTYFACTS_URL)).text());
|
||
const link = new Map<string, { pf: number | null; country: string }>();
|
||
for (const r of pf)
|
||
if (r.dataset_key === 'ches' || r.dataset_key === 'vparty' || r.dataset_key === 'manifesto')
|
||
link.set(`${r.dataset_key}:${r.dataset_party_id}`, {
|
||
pf: num(r.partyfacts_id),
|
||
country: r.country
|
||
});
|
||
const partiesById = new Map<string, Party>();
|
||
const upsert = (p: Partial<Party> & { party_id: string }) => {
|
||
const prev = partiesById.get(p.party_id);
|
||
if (!prev)
|
||
partiesById.set(p.party_id, {
|
||
iso3: null,
|
||
name: null,
|
||
abbrev: null,
|
||
family: null,
|
||
ches_id: null,
|
||
manifesto_id: null,
|
||
vparty_id: null,
|
||
partyfacts_id: null,
|
||
sort: null,
|
||
...p
|
||
});
|
||
else
|
||
for (const [k, v] of Object.entries(p))
|
||
if (v != null && (prev as Record<string, unknown>)[k] == null)
|
||
(prev as Record<string, unknown>)[k] = v;
|
||
};
|
||
const positions = new Map<string, Position>();
|
||
const put = (p: Position) => positions.set(`${p.party_id}|${p.year}|${p.source}|${p.dim}`, p);
|
||
|
||
// CHES
|
||
const ches = table(await (await http(CHES_URL)).text());
|
||
for (const r of ches) {
|
||
const chesId = num(r.party_id);
|
||
if (chesId == null) continue;
|
||
const l = link.get(`ches:${chesId}`);
|
||
const id = l?.pf != null ? `pf:${l.pf}` : `ches:${chesId}`;
|
||
const iso3 = CHES_COUNTRY[Number(r.country)] ?? l?.country ?? null;
|
||
if (!iso3) note(`CHES country ${r.country} unmapped`);
|
||
upsert({
|
||
party_id: id,
|
||
iso3,
|
||
abbrev: r.party || null,
|
||
family: CHES_FAMILY[Number(r.family)] ?? null,
|
||
ches_id: chesId,
|
||
manifesto_id: num(r.cmp_id),
|
||
partyfacts_id: l?.pf ?? null
|
||
});
|
||
for (const dim of CHES_DIMS)
|
||
if (r[dim] !== undefined)
|
||
put({
|
||
party_id: id,
|
||
year: Number(r.year),
|
||
source: 'ches',
|
||
dim,
|
||
value: num(r[dim]),
|
||
vote_share: num(r.vote)
|
||
});
|
||
}
|
||
const chesRows = positions.size;
|
||
|
||
// V-Party (RData → CSV via the stats Python venv)
|
||
let vpartyRows = 0;
|
||
try {
|
||
const file = join(tmpdir(), `vparty-${process.pid}.RData`);
|
||
await Bun.write(file, await (await http(VPARTY_URL)).arrayBuffer());
|
||
const proc = Bun.spawn([PYTHON, join(import.meta.dir, 'vparty-csv.py'), file], {
|
||
stdout: 'pipe',
|
||
stderr: 'pipe'
|
||
});
|
||
const csv = await new Response(proc.stdout).text();
|
||
const err = await new Response(proc.stderr).text();
|
||
if ((await proc.exited) !== 0) throw new Error(`vparty-csv.py failed: ${err.slice(0, 300)}`);
|
||
await Bun.file(file).delete();
|
||
for (const r of table(csv)) {
|
||
const vpId = num(r.v2paid);
|
||
if (vpId == null) continue;
|
||
const pfId = num(r.pf_party_id) ?? link.get(`vparty:${vpId}`)?.pf ?? null;
|
||
const id = pfId != null ? `pf:${pfId}` : `vp:${vpId}`;
|
||
upsert({
|
||
party_id: id,
|
||
iso3: r.country_text_id || null,
|
||
name: r.v2paenname || null,
|
||
abbrev: r.v2pashname || null,
|
||
vparty_id: vpId,
|
||
partyfacts_id: pfId,
|
||
ches_id: num(r.CHES_ID)
|
||
});
|
||
for (const dim of VPARTY_DIMS)
|
||
if (r[dim] !== undefined && r[dim] !== 'NA') {
|
||
put({
|
||
party_id: id,
|
||
year: Number(r.year),
|
||
source: 'vparty',
|
||
dim,
|
||
value: num(r[dim]),
|
||
vote_share: num(r.v2pavote)
|
||
});
|
||
vpartyRows++;
|
||
}
|
||
}
|
||
} catch (error) {
|
||
note(`V-Party skipped: ${error instanceof Error ? error.message : error}`);
|
||
}
|
||
|
||
// Names for CHES-only parties from Party Facts.
|
||
for (const r of pf)
|
||
if (r.dataset_key === 'ches') {
|
||
const id =
|
||
num(r.partyfacts_id) != null ? `pf:${r.partyfacts_id}` : `ches:${r.dataset_party_id}`;
|
||
const p = partiesById.get(id);
|
||
if (p && !p.name) p.name = r.name_english || r.name || null;
|
||
}
|
||
|
||
const list = [...partiesById.values()];
|
||
await sql.begin(async (tx) => {
|
||
await tx`delete from stats.wo_party_position where source in ('ches', 'vparty')`;
|
||
await tx`delete from stats.wo_party where party_id not like 'wo:%'`;
|
||
await insertChunks(tx, 'wo_party', list);
|
||
await insertChunks(tx, 'wo_party_position', [...positions.values()]);
|
||
// Link curated guide parties to Party Facts where exactly one data party in the same
|
||
// country has the same abbreviation (case-insensitive).
|
||
await tx`
|
||
update stats.wo_party w set partyfacts_id = m.partyfacts_id
|
||
from (
|
||
select c.party_id, min(d.partyfacts_id) as partyfacts_id
|
||
from stats.wo_party c
|
||
join stats.wo_party d on d.iso3 = c.iso3 and d.party_id not like 'wo:%'
|
||
and lower(d.abbrev) = lower(c.abbrev) and d.partyfacts_id is not null
|
||
where c.party_id like 'wo:%'
|
||
group by c.party_id having count(distinct d.partyfacts_id) = 1
|
||
) m
|
||
where w.party_id = m.party_id`;
|
||
});
|
||
log(`parties: ${list.length} parties, ${chesRows} CHES + ${vpartyRows} V-Party positions`);
|
||
return list.length + positions.size;
|
||
});
|
||
}
|