stats step 3: elections, party positions and opinion feeds (schema, collectors, timers); world-opinions data moved into stats.wo_*
All checks were successful
Deploy production / deploy (push) Successful in 1m12s
All checks were successful
Deploy production / deploy (push) Successful in 1m12s
This commit is contained in:
248
scripts/stats/parties.ts
Normal file
248
scripts/stats/parties.ts
Normal file
@@ -0,0 +1,248 @@
|
||||
// A15 party positions: CHES trend file (party means 1999–2024) and V-Party v2 (V-Dem, via the
|
||||
// vdemdata GitHub repo's RData), linked through the Party Facts crosswalk. Writes stats.wo_party
|
||||
// (data-driven rows; the curated guide parties keep their own `wo:` rows) and
|
||||
// stats.wo_party_position. Curated rows get a partyfacts_id when a match is unambiguous.
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { insertChunks, http, log, run, sql } from './common';
|
||||
import { parseCsv } from './owid';
|
||||
|
||||
const CHES_URL =
|
||||
'https://github.com/chesdata/chesdata.github.io/releases/download/ches-trend/1999-2024_CHES_dataset_meansV2.csv';
|
||||
const VPARTY_URL = 'https://github.com/vdeminstitute/vdemdata/raw/master/data/vparty.RData';
|
||||
const PARTYFACTS_URL = 'https://partyfacts.herokuapp.com/download/external-parties-csv/';
|
||||
const PYTHON = process.env.STATS_PYTHON ?? `${process.env.HOME}/.local/share/stats-py/bin/python`;
|
||||
|
||||
const CHES_COUNTRY: Record<number, string> = {
|
||||
1: 'BEL',
|
||||
2: 'DNK',
|
||||
3: 'DEU',
|
||||
4: 'GRC',
|
||||
5: 'ESP',
|
||||
6: 'FRA',
|
||||
7: 'IRL',
|
||||
8: 'ITA',
|
||||
10: 'NLD',
|
||||
11: 'GBR',
|
||||
12: 'PRT',
|
||||
13: 'AUT',
|
||||
14: 'FIN',
|
||||
16: 'SWE',
|
||||
20: 'BGR',
|
||||
21: 'CZE',
|
||||
22: 'EST',
|
||||
23: 'HUN',
|
||||
24: 'LVA',
|
||||
25: 'LTU',
|
||||
26: 'POL',
|
||||
27: 'ROU',
|
||||
28: 'SVK',
|
||||
29: 'SVN',
|
||||
31: 'HRV',
|
||||
37: 'MLT',
|
||||
38: 'LUX',
|
||||
40: 'CYP',
|
||||
9: 'NOR',
|
||||
35: 'NOR',
|
||||
36: 'CHE',
|
||||
34: 'TUR',
|
||||
45: 'ISL'
|
||||
};
|
||||
const CHES_FAMILY: Record<number, string> = {
|
||||
1: 'radical right',
|
||||
2: 'conservative',
|
||||
3: 'liberal',
|
||||
4: 'christian democratic',
|
||||
5: 'socialist',
|
||||
6: 'radical left',
|
||||
7: 'green',
|
||||
8: 'regionalist',
|
||||
9: 'no family',
|
||||
10: 'confessional',
|
||||
11: 'agrarian/centre'
|
||||
};
|
||||
const CHES_DIMS = ['lrgen', 'lrecon', 'galtan', 'eu_position', 'immigrate_policy'];
|
||||
const VPARTY_DIMS = [
|
||||
'v2xpa_popul',
|
||||
'v2xpa_antiplural',
|
||||
'v2pariglef',
|
||||
'v2paimmig',
|
||||
'v2palgbt',
|
||||
'v2parelig',
|
||||
'v2pawelf',
|
||||
'v2paculsup'
|
||||
];
|
||||
|
||||
type Party = {
|
||||
party_id: string;
|
||||
iso3: string | null;
|
||||
name: string | null;
|
||||
abbrev: string | null;
|
||||
family: string | null;
|
||||
ches_id: number | null;
|
||||
manifesto_id: number | null;
|
||||
vparty_id: number | null;
|
||||
partyfacts_id: number | null;
|
||||
sort: null;
|
||||
};
|
||||
type Position = {
|
||||
party_id: string;
|
||||
year: number;
|
||||
source: string;
|
||||
dim: string;
|
||||
value: number | null;
|
||||
vote_share: number | null;
|
||||
};
|
||||
|
||||
const table = (text: string) => {
|
||||
const [header, ...rows] = parseCsv(text);
|
||||
return rows
|
||||
.filter((r) => r.length >= header.length - 1)
|
||||
.map((r) => Object.fromEntries(header.map((h, i) => [h, r[i] ?? ''])));
|
||||
};
|
||||
const num = (s: string | undefined) =>
|
||||
s == null || s === '' || s === 'NA' ? null : Number.isFinite(Number(s)) ? Number(s) : null;
|
||||
|
||||
export async function parties(opts: { job: string }) {
|
||||
await run('ches', opts.job, async (note) => {
|
||||
// Party Facts: dataset ids → partyfacts ids (and countries).
|
||||
const pf = table(await (await http(PARTYFACTS_URL)).text());
|
||||
const link = new Map<string, { pf: number | null; country: string }>();
|
||||
for (const r of pf)
|
||||
if (r.dataset_key === 'ches' || r.dataset_key === 'vparty' || r.dataset_key === 'manifesto')
|
||||
link.set(`${r.dataset_key}:${r.dataset_party_id}`, {
|
||||
pf: num(r.partyfacts_id),
|
||||
country: r.country
|
||||
});
|
||||
const partiesById = new Map<string, Party>();
|
||||
const upsert = (p: Partial<Party> & { party_id: string }) => {
|
||||
const prev = partiesById.get(p.party_id);
|
||||
if (!prev)
|
||||
partiesById.set(p.party_id, {
|
||||
iso3: null,
|
||||
name: null,
|
||||
abbrev: null,
|
||||
family: null,
|
||||
ches_id: null,
|
||||
manifesto_id: null,
|
||||
vparty_id: null,
|
||||
partyfacts_id: null,
|
||||
sort: null,
|
||||
...p
|
||||
});
|
||||
else
|
||||
for (const [k, v] of Object.entries(p))
|
||||
if (v != null && (prev as Record<string, unknown>)[k] == null)
|
||||
(prev as Record<string, unknown>)[k] = v;
|
||||
};
|
||||
const positions = new Map<string, Position>();
|
||||
const put = (p: Position) => positions.set(`${p.party_id}|${p.year}|${p.source}|${p.dim}`, p);
|
||||
|
||||
// CHES
|
||||
const ches = table(await (await http(CHES_URL)).text());
|
||||
for (const r of ches) {
|
||||
const chesId = num(r.party_id);
|
||||
if (chesId == null) continue;
|
||||
const l = link.get(`ches:${chesId}`);
|
||||
const id = l?.pf != null ? `pf:${l.pf}` : `ches:${chesId}`;
|
||||
const iso3 = CHES_COUNTRY[Number(r.country)] ?? l?.country ?? null;
|
||||
if (!iso3) note(`CHES country ${r.country} unmapped`);
|
||||
upsert({
|
||||
party_id: id,
|
||||
iso3,
|
||||
abbrev: r.party || null,
|
||||
family: CHES_FAMILY[Number(r.family)] ?? null,
|
||||
ches_id: chesId,
|
||||
manifesto_id: num(r.cmp_id),
|
||||
partyfacts_id: l?.pf ?? null
|
||||
});
|
||||
for (const dim of CHES_DIMS)
|
||||
if (r[dim] !== undefined)
|
||||
put({
|
||||
party_id: id,
|
||||
year: Number(r.year),
|
||||
source: 'ches',
|
||||
dim,
|
||||
value: num(r[dim]),
|
||||
vote_share: num(r.vote)
|
||||
});
|
||||
}
|
||||
const chesRows = positions.size;
|
||||
|
||||
// V-Party (RData → CSV via the stats Python venv)
|
||||
let vpartyRows = 0;
|
||||
try {
|
||||
const file = join(tmpdir(), `vparty-${process.pid}.RData`);
|
||||
await Bun.write(file, await (await http(VPARTY_URL)).arrayBuffer());
|
||||
const proc = Bun.spawn([PYTHON, join(import.meta.dir, 'vparty-csv.py'), file], {
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe'
|
||||
});
|
||||
const csv = await new Response(proc.stdout).text();
|
||||
const err = await new Response(proc.stderr).text();
|
||||
if ((await proc.exited) !== 0) throw new Error(`vparty-csv.py failed: ${err.slice(0, 300)}`);
|
||||
await Bun.file(file).delete();
|
||||
for (const r of table(csv)) {
|
||||
const vpId = num(r.v2paid);
|
||||
if (vpId == null) continue;
|
||||
const pfId = num(r.pf_party_id) ?? link.get(`vparty:${vpId}`)?.pf ?? null;
|
||||
const id = pfId != null ? `pf:${pfId}` : `vp:${vpId}`;
|
||||
upsert({
|
||||
party_id: id,
|
||||
iso3: r.country_text_id || null,
|
||||
name: r.v2paenname || null,
|
||||
abbrev: r.v2pashname || null,
|
||||
vparty_id: vpId,
|
||||
partyfacts_id: pfId,
|
||||
ches_id: num(r.CHES_ID)
|
||||
});
|
||||
for (const dim of VPARTY_DIMS)
|
||||
if (r[dim] !== undefined && r[dim] !== 'NA') {
|
||||
put({
|
||||
party_id: id,
|
||||
year: Number(r.year),
|
||||
source: 'vparty',
|
||||
dim,
|
||||
value: num(r[dim]),
|
||||
vote_share: num(r.v2pavote)
|
||||
});
|
||||
vpartyRows++;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
note(`V-Party skipped: ${error instanceof Error ? error.message : error}`);
|
||||
}
|
||||
|
||||
// Names for CHES-only parties from Party Facts.
|
||||
for (const r of pf)
|
||||
if (r.dataset_key === 'ches') {
|
||||
const id =
|
||||
num(r.partyfacts_id) != null ? `pf:${r.partyfacts_id}` : `ches:${r.dataset_party_id}`;
|
||||
const p = partiesById.get(id);
|
||||
if (p && !p.name) p.name = r.name_english || r.name || null;
|
||||
}
|
||||
|
||||
const list = [...partiesById.values()];
|
||||
await sql.begin(async (tx) => {
|
||||
await tx`delete from stats.wo_party_position where source in ('ches', 'vparty')`;
|
||||
await tx`delete from stats.wo_party where party_id not like 'wo:%'`;
|
||||
await insertChunks(tx, 'wo_party', list);
|
||||
await insertChunks(tx, 'wo_party_position', [...positions.values()]);
|
||||
// Link curated guide parties to Party Facts where exactly one data party in the same
|
||||
// country has the same abbreviation (case-insensitive).
|
||||
await tx`
|
||||
update stats.wo_party w set partyfacts_id = m.partyfacts_id
|
||||
from (
|
||||
select c.party_id, min(d.partyfacts_id) as partyfacts_id
|
||||
from stats.wo_party c
|
||||
join stats.wo_party d on d.iso3 = c.iso3 and d.party_id not like 'wo:%'
|
||||
and lower(d.abbrev) = lower(c.abbrev) and d.partyfacts_id is not null
|
||||
where c.party_id like 'wo:%'
|
||||
group by c.party_id having count(distinct d.partyfacts_id) = 1
|
||||
) m
|
||||
where w.party_id = m.party_id`;
|
||||
});
|
||||
log(`parties: ${list.length} parties, ${chesRows} CHES + ${vpartyRows} V-Party positions`);
|
||||
return list.length + positions.size;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user