stats step 3: elections, party positions and opinion feeds (schema, collectors, timers); world-opinions data moved into stats.wo_*

2026-10-25 22:16:25 +00:00
parent 030f3c08ee
commit 42c9351ebd
37 changed files with 75721 additions and 892 deletions
--- a/scripts/stats/gdelt.ts
+++ b/scripts/stats/gdelt.ts
@@ -0,0 +1,86 @@
+// A16 GDELT DOC 2.0 "timelinevol": daily share of monitored news coverage (by source country)
+// that mentions a GKG theme, last 30 days, upserted nightly. GDELT asks for at most one request
+// every 5 seconds, so the loop is deliberately slow (~12 minutes). GDELT data is open, cite GDELT.
+import { http, insertChunks, run, sql } from './common';
+
+export const GDELT_COUNTRIES: Record<string, string> = {
+	netherlands: 'NLD',
+	unitedstates: 'USA',
+	southkorea: 'KOR',
+	germany: 'DEU',
+	france: 'FRA',
+	unitedkingdom: 'GBR',
+	belgium: 'BEL',
+	japan: 'JPN',
+	india: 'IND',
+	brazil: 'BRA'
+};
+export const GDELT_THEMES = [
+	'ECON_INFLATION',
+	'ECON_HOUSING_PRICES',
+	'UNEMPLOYMENT',
+	'IMMIGRATION',
+	'REFUGEES',
+	'ENV_CLIMATECHANGE',
+	'TERROR',
+	'PROTEST',
+	'CORRUPTION',
+	'ELECTION',
+	'MILITARY',
+	'HEALTH_PANDEMIC'
+];
+
+export async function gdelt(opts: { job: string; timespan?: string }) {
+	await run('gdelt', opts.job, async (note) => {
+		let rows = 0;
+		const bad = new Set<string>();
+		for (const [name, iso3] of Object.entries(GDELT_COUNTRIES))
+			for (const theme of GDELT_THEMES) {
+				if (bad.has(theme)) continue;
+				const q = encodeURIComponent(`theme:${theme} sourcecountry:${name}`);
+				const url = `https://api.gdeltproject.org/api/v2/doc/doc?query=${q}&mode=timelinevol&format=json&timespan=${opts.timespan ?? '30d'}`;
+				let text: string;
+				try {
+					text = await (await http(url, {}, 3)).text();
+				} catch (error) {
+					note(`${theme}/${name}: ${error instanceof Error ? error.message : error}`);
+					continue;
+				}
+				let data: { timeline?: { data: { date: string; value: number }[] }[] };
+				try {
+					data = JSON.parse(text);
+				} catch {
+					// GDELT answers invalid queries with a plain-text message.
+					note(`${theme}: ${text.slice(0, 120)}`);
+					if (/theme|invalid/i.test(text)) bad.add(theme);
+					continue;
+				}
+				const points = (data.timeline?.[0]?.data ?? []).map((p) => ({
+					date: `${p.date.slice(0, 4)}-${p.date.slice(4, 6)}-${p.date.slice(6, 8)}`,
+					country: iso3,
+					theme,
+					vol_pct: p.value
+				}));
+				// Hourly resolution (short timespans): average per day.
+				const byDay = new Map<string, { sum: number; n: number; row: (typeof points)[number] }>();
+				for (const p of points) {
+					const d = byDay.get(p.date) ?? { sum: 0, n: 0, row: p };
+					d.sum += p.vol_pct;
+					d.n++;
+					byDay.set(p.date, d);
+				}
+				const daily = [...byDay.values()].map((d) => ({ ...d.row, vol_pct: d.sum / d.n }));
+				if (!daily.length) continue;
+				await sql.begin(async (tx) => {
+					await insertChunks(
+						tx,
+						'gdelt_theme_vol',
+						daily,
+						'on conflict (date, country, theme) do update set vol_pct = excluded.vol_pct'
+					);
+				});
+				rows += daily.length;
+			}
+		return rows;
+	});
+}