stats step 3: elections, party positions and opinion feeds (schema, collectors, timers); world-opinions data moved into stats.wo_*

scripts/stats/whr.ts

@@ -0,0 +1,124 @@
+// A16 World Happiness Report: Figure 2.1 data (life evaluation, 3-year average, and the
+// explained components), all years 2011–latest. Tries the newest edition first.
+// © Wellbeing Research Centre / Gallup World Poll data, reuse with attribution.
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { http, insertChunks, log, run, sql } from './common';
+import { readXlsx, type Cell } from './xlsx';
+
+const NAME_ISO3: Record<string, string | null> = {
+	'Côte d’Ivoire': 'CIV',
+	"Côte d'Ivoire": 'CIV',
+	'Ivory Coast': 'CIV',
+	'DR Congo': 'COD',
+	'Hong Kong SAR of China': 'HKG',
+	'Hong Kong S.A.R. of China': 'HKG',
+	Kosovo: 'XKX',
+	'Lao PDR': 'LAO',
+	Laos: 'LAO',
+	'Republic of Korea': 'KOR',
+	'South Korea': 'KOR',
+	'Republic of Moldova': 'MDA',
+	'Russian Federation': 'RUS',
+	'State of Palestine': 'PSE',
+	Swaziland: 'SWZ',
+	Eswatini: 'SWZ',
+	'Taiwan Province of China': 'TWN',
+	Türkiye: 'TUR',
+	Turkiye: 'TUR',
+	'Viet Nam': 'VNM',
+	'North Cyprus': null,
+	'Somaliland Region': null,
+	Czechia: 'CZE',
+	'Congo (Brazzaville)': 'COG',
+	'Congo (Kinshasa)': 'COD'
+};
+
+const n = (c: Cell | undefined) =>
+	typeof c === 'number'
+		? c
+		: c == null || c === ''
+			? null
+			: Number.isFinite(Number(c))
+				? Number(c)
+				: null;
+
+export async function whr(opts: { job: string }) {
+	await run('whr', opts.job, async (note) => {
+		const yy = new Date().getFullYear() % 100;
+		let file: string | null = null;
+		for (const e of [yy + 1, yy, yy - 1]) {
+			try {
+				const res = await http(
+					`https://files.worldhappiness.report/WHR${e}_Data_Figure_2.1.xlsx`,
+					{},
+					1
+				);
+				file = join(tmpdir(), `whr-${process.pid}.xlsx`);
+				await Bun.write(file, await res.arrayBuffer());
+				log(`whr: edition 20${e}`);
+				break;
+			} catch {
+				// try an older edition
+			}
+		}
+		if (!file) throw new Error('no WHR Figure 2.1 file found');
+		const [sheet] = await readXlsx(file);
+		await Bun.file(file).delete();
+		const [header, ...rows] = sheet.rows;
+		const col = (re: RegExp) => header.findIndex((h) => re.test(String(h ?? '')));
+		const c = {
+			year: col(/^Year/),
+			rank: col(/^Rank/),
+			country: col(/^Country/),
+			ladder: col(/^Life evaluation|^Ladder/),
+			lower: col(/^Lower/),
+			upper: col(/^Upper/),
+			gdp: col(/GDP/),
+			social: col(/Social/),
+			health: col(/Healthy/),
+			freedom: col(/Freedom/),
+			generosity: col(/Generosity/),
+			corruption: col(/corruption/i),
+			dystopia: col(/Dystopia/)
+		};
+		if (c.year < 0 || c.country < 0 || c.ladder < 0)
+			throw new Error(`WHR columns changed: ${header.join('|')}`);
+		const iso = new Map(
+			(await sql`select distinct country, iso3 from stats.owid_energy where length(iso3) = 3`).map(
+				(r) => [r.country as string, r.iso3 as string]
+			)
+		);
+		const unmapped = new Set<string>();
+		const out = rows
+			.filter((r) => n(r[c.year]) && r[c.country])
+			.map((r) => {
+				const country = String(r[c.country]);
+				const iso3 = country in NAME_ISO3 ? NAME_ISO3[country] : (iso.get(country) ?? null);
+				if (!iso3 && !(country in NAME_ISO3)) unmapped.add(country);
+				const v = (k: keyof typeof c) => (c[k] >= 0 ? n(r[c[k]]) : null);
+				return {
+					year: v('year')!,
+					country,
+					iso3,
+					rank: v('rank'),
+					ladder: v('ladder'),
+					lower: v('lower'),
+					upper: v('upper'),
+					gdp: v('gdp'),
+					social: v('social'),
+					health: v('health'),
+					freedom: v('freedom'),
+					generosity: v('generosity'),
+					corruption: v('corruption'),
+					dystopia: v('dystopia')
+				};
+			});
+		if (unmapped.size) note(`no ISO3 for: ${[...unmapped].join(', ')}`);
+		await sql.begin(async (tx) => {
+			await tx`delete from stats.whr_score`;
+			await insertChunks(tx, 'whr_score', out);
+		});
+		return out.length;
+	});
+}