stats step 3: elections, party positions and opinion feeds (schema, collectors, timers); world-opinions data moved into stats.wo_*

deploy/stats-collector/README.md

@@ -17,6 +17,13 @@ statistics into the `stats` schema with `scripts/stats-fetch.ts`:
 | `ustreasury`     | `stats-ustreasury.timer`     | daily 23:25                 | US Treasury MTS receipts/outlays + Debt to the Penny (last 400 days; `--backfill` all)                                                                                                                                                                                            |
 | `usaspending`    | `stats-usaspending.timer`    | Sundays 06:10               | USAspending budgetary resources per agency and fiscal year                                                                                                                                                                                                                        |
 | `eponline`       | `stats-eponline.timer`       | 3rd of the month 05:20      | EP-Online monthly total file (RVO energy labels): homes per label for NL, Gouda and Gouda PC4s; aggregates only. Needs `EP_ONLINE_API_KEY` in `~/.config/stats/ep-online.env` (mode 600), which the runner exports for this source only                                           |
+| `elections`      | `stats-elections.timer`      | 15th of the month 04:50     | A14: Kiesraad Tweede Kamer results 2010→ (data.overheid.nl CSV + EML zips read with HTTP ranges; national, municipality, Gouda polling stations), CBS 37278 (1918–2006), MIT US presidential by state. Finished elections are skipped unless `--force`                            |
+| `parties`        | `stats-parties.timer`        | quarterly, 16th 04:40       | A15: CHES trend file + V-Party (RData via `~/.local/share/stats-py`, pyreadr) linked with Party Facts into `wo_party`/`wo_party_position`                                                                                                                                         |
+| `wikipolls`      | `stats-wikipolls.timer`      | daily 08:10                 | A16: Wikipedia poll tables (NL seats, Trump approval, Lee Jae-myung approval, KR party polls); wikitext snapshot per revision                                                                                                                                                     |
+| `gtrends`        | `stats-gtrends.timer`        | daily 21:45                 | A16: Google Trends trending-now RSS for NL, US, KR                                                                                                                                                                                                                                |
+| `gdelt`          | `stats-gdelt.timer`          | nightly 02:30               | A16: GDELT DOC timelinevol, 10 countries × 12 themes, last 30 days, ≥5.5 s between requests                                                                                                                                                                                       |
+| `eurobarometer`  | `stats-eurobarometer.timer`  | 20th of the month 05:35     | A16: Standard Eurobarometer volume A xlsx (QA3/QA4/QA5 most important issues, trust in institutions); new waves only                                                                                                                                                              |
+| `whr`            | `stats-whr.timer`            | 21st of the month 05:10     | A16: World Happiness Report Figure 2.1 (life evaluation and components, 2011→)                                                                                                                                                                                                    |
 | `cbscw`          | `stats-cbscw.timer`          | Tuesdays 05:50              | CBS Curaçao: scrapes the topic pages (cbs.cw), matches each table's file by regex, re-imports only when its sha256 changes (`stats.cw_cbs_file`); CPI from PDF (pdfjs), GDP, government finance, population since 1900, LFS, tourism from xlsx                                    |
 | `cwintl`         | `stats-cwintl.timer`         | Wednesdays 05:50            | Curaçao from international APIs: ILOSTAT SDMX (labour, with retries), OWID grapher CSVs (population, life expectancy, median age, CO₂), UN SDG area 531, UN Comtrade public preview reporter 531. EU trade with Curaçao (Eurostat `ext_lt_maineu`) runs with the `eurostat` timer |
 | –                | –                            | manual                      | CBCS (centralbank.cw) blocks scripts with a Cloudflare challenge: download the BoP xlsx by hand and load it into `stats.cw_cbcs_bop` (period, item, mln_ang)                                                                                                                      |

deploy/stats-collector/stats-elections.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: Kiesraad TK results, CBS 37278 history, MIT US presidential (new elections only)
+
+[Timer]
+OnCalendar=*-*-15 04:50:00 Europe/Amsterdam
+RandomizedDelaySec=20min
+Persistent=true
+Unit=stats-collector@elections.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-eurobarometer.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: Standard Eurobarometer volume A, new waves (monthly check)
+
+[Timer]
+OnCalendar=*-*-20 05:35:00 Europe/Amsterdam
+RandomizedDelaySec=20min
+Persistent=true
+Unit=stats-collector@eurobarometer.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-gdelt.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: GDELT theme coverage, slow loop at 1 request per 5.5 s (nightly)
+
+[Timer]
+OnCalendar=*-*-* 02:30:00 Europe/Amsterdam
+RandomizedDelaySec=20min
+Persistent=true
+Unit=stats-collector@gdelt.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-gtrends.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: Google Trends trending-now RSS for NL, US, KR (daily)
+
+[Timer]
+OnCalendar=*-*-* 21:45:00 Europe/Amsterdam
+RandomizedDelaySec=20min
+Persistent=true
+Unit=stats-collector@gtrends.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-parties.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: CHES + V-Party party positions via Party Facts (quarterly)
+
+[Timer]
+OnCalendar=*-01,04,07,10-16 04:40:00 Europe/Amsterdam
+RandomizedDelaySec=20min
+Persistent=true
+Unit=stats-collector@parties.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-whr.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: World Happiness Report Figure 2.1 (monthly check)
+
+[Timer]
+OnCalendar=*-*-21 05:10:00 Europe/Amsterdam
+RandomizedDelaySec=20min
+Persistent=true
+Unit=stats-collector@whr.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-wikipolls.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: Wikipedia opinion-poll tables (daily)
+
+[Timer]
+OnCalendar=*-*-* 08:10:00 Europe/Amsterdam
+RandomizedDelaySec=20min
+Persistent=true
+Unit=stats-collector@wikipolls.service
+
+[Install]
+WantedBy=timers.target

drizzle/20261025220251_stats_step3/migration.sql

@@ -0,0 +1,193 @@
+CREATE TABLE "stats"."eb_item" (
+	"wave" text,
+	"fieldwork_end" date,
+	"topic" text,
+	"qcode" text NOT NULL,
+	"item" text,
+	"country" text,
+	"share" double precision,
+	CONSTRAINT "eb_item_pkey" PRIMARY KEY("wave","topic","item","country")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."elec_election" (
+	"election_id" text PRIMARY KEY,
+	"country" char(3) NOT NULL,
+	"kind" text NOT NULL,
+	"date" date NOT NULL,
+	"name" text NOT NULL,
+	"source_id" text NOT NULL,
+	"total_seats" integer
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."elec_result" (
+	"election_id" text,
+	"level" text,
+	"area_code" text,
+	"area_name" text,
+	"party" text,
+	"votes" bigint,
+	"seats" integer,
+	CONSTRAINT "elec_result_pkey" PRIMARY KEY("election_id","level","area_code","party")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."elec_turnout" (
+	"election_id" text,
+	"level" text,
+	"area_code" text,
+	"area_name" text,
+	"electorate" bigint,
+	"ballots" bigint,
+	"valid" bigint,
+	"blank" bigint,
+	"invalid" bigint,
+	CONSTRAINT "elec_turnout_pkey" PRIMARY KEY("election_id","level","area_code")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."gdelt_theme_vol" (
+	"date" date,
+	"country" char(3),
+	"theme" text,
+	"vol_pct" double precision,
+	CONSTRAINT "gdelt_theme_vol_pkey" PRIMARY KEY("date","country","theme")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."gt_trending" (
+	"geo" text,
+	"date" date,
+	"rank" integer NOT NULL,
+	"query" text,
+	"approx_traffic" integer,
+	"pub_date" timestamp with time zone,
+	"news" text,
+	CONSTRAINT "gt_trending_pkey" PRIMARY KEY("geo","date","query")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."op_poll" (
+	"series" text,
+	"poll_key" text,
+	"fieldwork_end" date NOT NULL,
+	"pollster" text NOT NULL,
+	"sample" integer,
+	"option" text,
+	"value" double precision,
+	"seen_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "op_poll_pkey" PRIMARY KEY("series","poll_key","option")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."op_wiki_snapshot" (
+	"page" text,
+	"revid" bigint,
+	"fetched_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"wikitext" text NOT NULL,
+	CONSTRAINT "op_wiki_snapshot_pkey" PRIMARY KEY("page","revid")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."whr_score" (
+	"year" integer,
+	"country" text,
+	"iso3" char(3),
+	"rank" integer,
+	"ladder" double precision,
+	"lower" double precision,
+	"upper" double precision,
+	"gdp" double precision,
+	"social" double precision,
+	"health" double precision,
+	"freedom" double precision,
+	"generosity" double precision,
+	"corruption" double precision,
+	"dystopia" double precision,
+	CONSTRAINT "whr_score_pkey" PRIMARY KEY("year","country")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."wo_compass_override" (
+	"party_id" text PRIMARY KEY,
+	"x" double precision NOT NULL,
+	"y" double precision NOT NULL,
+	"note" text
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."wo_country" (
+	"iso3" char(3) PRIMARY KEY,
+	"id" text NOT NULL UNIQUE,
+	"iso_num" text NOT NULL,
+	"name" text NOT NULL,
+	"region" text NOT NULL,
+	"liberal_meaning" text,
+	"main_divide" text,
+	"note" text,
+	"profile" jsonb,
+	"sort" integer NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."wo_cultural_map" (
+	"iso3" char(3),
+	"wave" text,
+	"year_from" integer,
+	"year_to" integer,
+	"trad_secular" double precision,
+	"surv_selfexp" double precision,
+	"n" integer,
+	"source" text,
+	CONSTRAINT "wo_cultural_map_pkey" PRIMARY KEY("iso3","wave")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."wo_narrative" (
+	"sort" integer PRIMARY KEY,
+	"event" text NOT NULL,
+	"years" text NOT NULL,
+	"views" jsonb NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."wo_party" (
+	"party_id" text PRIMARY KEY,
+	"iso3" char(3),
+	"name" text,
+	"abbrev" text,
+	"family" text,
+	"ches_id" integer,
+	"manifesto_id" integer,
+	"vparty_id" integer,
+	"partyfacts_id" integer,
+	"sort" integer
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."wo_party_position" (
+	"party_id" text,
+	"year" integer,
+	"source" text,
+	"dim" text,
+	"value" double precision,
+	"vote_share" double precision,
+	CONSTRAINT "wo_party_position_pkey" PRIMARY KEY("party_id","year","source","dim")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."wo_source" (
+	"sort" integer PRIMARY KEY,
+	"name" text NOT NULL,
+	"href" text NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."wo_value_score" (
+	"iso3" char(3),
+	"wave" text,
+	"item_code" text,
+	"item_label" text,
+	"stat" text,
+	"value" double precision,
+	"n" integer,
+	CONSTRAINT "wo_value_score_pkey" PRIMARY KEY("iso3","wave","item_code","stat")
+);
+--> statement-breakpoint
+CREATE INDEX "elec_result_area_idx" ON "stats"."elec_result" ("level","area_code");--> statement-breakpoint
+CREATE INDEX "op_poll_series_date_idx" ON "stats"."op_poll" ("series","fieldwork_end");--> statement-breakpoint
+CREATE INDEX "wo_party_iso3_idx" ON "stats"."wo_party" ("iso3");--> statement-breakpoint
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA stats TO personal_app;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
+    GRANT SELECT ON ALL TABLES IN SCHEMA stats TO personal_ro;
+  END IF;
+END $$;

drizzle/20261025220252_stats_wo_seed/migration.sql

@@ -0,0 +1,239 @@
+-- Seed of /projects/world-opinions, generated from src/lib/world-opinions/data.ts (commit 030f3c0).
+-- Curated text and compass placements copied exactly; the DB is the source of truth from here on.
+INSERT INTO "stats"."wo_country" ("iso3","id","iso_num","name","region","liberal_meaning","main_divide","note","profile","sort") VALUES
+('SWE', 'se', '752', 'Sweden', 'nordics', 'market', 'economy', NULL, '{"center": "High-trust, high-tax welfare state; very secular and individualist", "fights": ["immigration", "gang violence", "private welfare", "NATO"], "timeline": [{"year": "1814", "label": "last war fought"}, {"year": "1932", "label": "Social Democrats'' era"}, {"year": "1939", "label": "neutral in WWII"}, {"year": "2015", "label": "record asylum intake"}, {"year": "2024", "label": "joins NATO"}], "blindSpots": ["consensus makes disagreement taboo", "neutrality myth (iron to Germany)"]}'::jsonb, 0),
+('DNK', 'dk', '208', 'Denmark', 'nordics', 'market', 'economy', '"Venstre" (literally "Left") is the liberal, center-right party', NULL, 1),
+('NOR', 'no', '578', 'Norway', 'nordics', 'market', 'economy', NULL, NULL, 2),
+('FIN', 'fi', '246', 'Finland', 'nordics', 'market', 'economy', NULL, NULL, 3),
+('NLD', 'nl', '528', 'Netherlands', 'western-europe', 'market', 'economy', 'Liberal VVD is the main center-right party', '{"center": "Coalition consensus (\"polder model\") around a strong welfare state", "fights": ["immigration", "housing", "nitrogen & farmers", "EU"], "timeline": [{"year": "1600s", "label": "Golden Age, trade empire"}, {"year": "1940", "label": "German occupation"}, {"year": "1949", "label": "loses Indonesia"}, {"year": "1960s", "label": "pillars break, secularization"}, {"year": "2002", "label": "Fortuyn, immigration debate"}], "blindSpots": ["tolerance self-image vs discrimination", "colonial past taught thinly"]}'::jsonb, 4),
+('BEL', 'be', '056', 'Belgium', 'western-europe', 'market', 'economy', 'Language (Flemish vs French-speaking) splits every party family', NULL, 5),
+('DEU', 'de', '276', 'Germany', 'western-europe', 'market', 'economy', 'Liberal FDP = pro-market, low taxes', '{"center": "Social market economy, fiscal caution, \"never again\"", "fights": ["migration", "energy & climate", "East vs West", "debt brake"], "timeline": [{"year": "1923", "label": "hyperinflation"}, {"year": "1933", "label": "Nazi dictatorship"}, {"year": "1949", "label": "split + economic miracle"}, {"year": "1990", "label": "reunification"}, {"year": "2015", "label": "refugee crisis"}], "blindSpots": ["debt as moral failing", "Russian gas as \"just trade\""]}'::jsonb, 6),
+('AUT', 'at', '040', 'Austria', 'western-europe', 'market', 'economy', NULL, NULL, 7),
+('CHE', 'ch', '756', 'Switzerland', 'western-europe', 'market', 'economy', NULL, NULL, 8),
+('GBR', 'gb', '826', 'United Kingdom', 'western-europe', 'market', 'economy', 'Liberal Democrats sit in the center; Brexit cut across old lines', '{"center": "Pragmatic and market-friendly, but the NHS is sacred", "fights": ["Brexit", "class", "North vs South", "Scotland"], "timeline": [{"year": "1900", "label": "largest empire in history"}, {"year": "1940", "label": "\"stood alone\""}, {"year": "1948", "label": "NHS founded"}, {"year": "1979", "label": "Thatcher"}, {"year": "2016", "label": "Brexit vote"}], "blindSpots": ["empire taught selectively", "overrates the \"special relationship\""]}'::jsonb, 9),
+('IRL', 'ie', '372', 'Ireland', 'western-europe', 'market', 'history', 'The two big parties descend from opposite sides of the 1922 civil war', NULL, 10),
+('FRA', 'fr', '250', 'France', 'western-europe', 'market', 'economy', '"Libéral" means pro-market, often used as a criticism', '{"center": "Strong central state, laïcité, protest as normal politics", "fights": ["pensions", "Paris vs periphery", "Islam & laïcité", "EU"], "timeline": [{"year": "1789", "label": "Revolution invents left/right"}, {"year": "1940", "label": "defeat and Vichy"}, {"year": "1962", "label": "loses Algerian war"}, {"year": "1968", "label": "May ''68"}, {"year": "2018", "label": "Yellow Vests"}], "blindSpots": ["colorblind universalism hides racism", "influence in former African colonies"]}'::jsonb, 11),
+('ESP', 'es', '724', 'Spain', 'western-europe', 'market', 'economy', 'Regional nationalism (Catalonia, Basque Country) is a second axis', NULL, 12),
+('PRT', 'pt', '620', 'Portugal', 'western-europe', 'market', 'economy', NULL, NULL, 13),
+('ITA', 'it', '380', 'Italy', 'western-europe', 'market', 'economy', NULL, NULL, 14),
+('GRC', 'gr', '300', 'Greece', 'eastern-europe', 'market', 'economy', NULL, NULL, 15),
+('USA', 'us', '840', 'United States', 'north-america', 'left', 'culture', '"Liberal" = Democrat; the Democrats sit near Europe''s center economically', '{"center": "Individual liberty and suspicion of government, on both sides", "fights": ["abortion", "guns", "race", "immigration", "religion"], "timeline": [{"year": "1776", "label": "revolt against a distant state"}, {"year": "1861", "label": "civil war over slavery"}, {"year": "1945", "label": "wins WWII, superpower"}, {"year": "1964", "label": "Civil Rights Act"}, {"year": "1991", "label": "wins the Cold War"}], "blindSpots": ["assumes its left/right is universal", "partisan media bubbles", "little view of other systems"]}'::jsonb, 16),
+('CAN', 'ca', '124', 'Canada', 'north-america', 'left', 'economy', NULL, NULL, 17),
+('MEX', 'mx', '484', 'Mexico', 'latin-america', 'market', 'economy', NULL, NULL, 18),
+('BRA', 'br', '076', 'Brazil', 'latin-america', 'market', 'economy', 'Bolsonaro''s party is literally called "Partido Liberal"', '{"center": "Inequality makes redistribution central; religious society", "fights": ["Lula vs Bolsonaro", "crime", "Amazon vs agribusiness"], "timeline": [{"year": "1888", "label": "last to abolish slavery"}, {"year": "1964", "label": "military coup"}, {"year": "1985", "label": "democracy returns"}, {"year": "1994", "label": "ends hyperinflation"}, {"year": "2014", "label": "Car Wash scandal"}], "blindSpots": ["\"racial democracy\" myth", "WhatsApp information bubbles"]}'::jsonb, 19),
+('ARG', 'ar', '032', 'Argentina', 'latin-america', 'market', 'economy', 'President Milei calls himself a "liberal-libertarian"', NULL, 20),
+('CHL', 'cl', '152', 'Chile', 'latin-america', 'market', 'economy', NULL, NULL, 21),
+('COL', 'co', '170', 'Colombia', 'latin-america', NULL, 'economy', NULL, NULL, 22),
+('PER', 'pe', '604', 'Peru', 'latin-america', 'market', 'economy', NULL, NULL, 23),
+('VEN', 've', '862', 'Venezuela', 'latin-america', NULL, 'none', NULL, NULL, 24),
+('AUS', 'au', '036', 'Australia', 'oceania', 'market', 'economy', 'The Liberal Party is the main conservative party', NULL, 25),
+('NZL', 'nz', '554', 'New Zealand', 'oceania', 'market', 'economy', NULL, NULL, 26),
+('POL', 'pl', '616', 'Poland', 'eastern-europe', 'western', 'culture', NULL, '{"center": "Pro-NATO and wary of Russia across all parties", "fights": ["Catholic vs liberal", "abortion", "rule of law", "EU"], "timeline": [{"year": "1795", "label": "erased by partition"}, {"year": "1939", "label": "invaded by Germany and USSR"}, {"year": "1980", "label": "Solidarity movement"}, {"year": "1989", "label": "communism falls"}, {"year": "2004", "label": "joins the EU"}], "blindSpots": ["victimhood crowds out dark chapters", "distrust of Germany"]}'::jsonb, 27),
+('CZE', 'cz', '203', 'Czechia', 'eastern-europe', 'western', 'culture', NULL, NULL, 28),
+('HUN', 'hu', '348', 'Hungary', 'eastern-europe', 'western', 'culture', 'Orbán calls his model "illiberal democracy"', NULL, 29),
+('ROU', 'ro', '642', 'Romania', 'eastern-europe', 'western', 'culture', NULL, NULL, 30),
+('SRB', 'rs', '688', 'Serbia', 'eastern-europe', 'western', 'history', NULL, NULL, 31),
+('UKR', 'ua', '804', 'Ukraine', 'eastern-europe', NULL, 'history', 'Politics long split between pro-Western and pro-Russian orientation', NULL, 32),
+('RUS', 'ru', '643', 'Russia', 'eastern-europe', 'western', 'none', '"Liberal" = the pro-Western opposition, mostly in exile or prison', '{"center": "Strong state and great-power status; opposition repressed", "fights": ["largely suppressed in public"], "timeline": [{"year": "1812", "label": "Napoleon invades"}, {"year": "1941", "label": "Nazi invasion, ~27M dead"}, {"year": "1991", "label": "USSR collapses"}, {"year": "1990s", "label": "chaos and poverty"}, {"year": "2022", "label": "full invasion of Ukraine"}], "blindSpots": ["state-controlled media", "neighbors seen as sphere of influence", "Soviet crimes downplayed"]}'::jsonb, 33),
+('KAZ', 'kz', '398', 'Kazakhstan', 'eastern-europe', NULL, 'none', NULL, NULL, 34),
+('TUR', 'tr', '792', 'Turkey', 'middle-east', 'western', 'culture', 'Main divide: religious-conservative vs secular', '{"center": "National pride, strong state, balancing West and region", "fights": ["religious vs secular", "Kurdish question", "inflation"], "timeline": [{"year": "1915", "label": "Armenian genocide (denied)"}, {"year": "1920", "label": "Sèvres, fear of partition"}, {"year": "1923", "label": "Atatürk''s secular republic"}, {"year": "1980", "label": "military coup"}, {"year": "2016", "label": "failed coup"}], "blindSpots": ["genocide denial", "restricted press", "Western criticism read as hostility"]}'::jsonb, 35),
+('IRN', 'ir', '364', 'Iran', 'middle-east', 'western', 'none', NULL, NULL, 36),
+('IRQ', 'iq', '368', 'Iraq', 'middle-east', NULL, 'culture', NULL, NULL, 37),
+('EGY', 'eg', '818', 'Egypt', 'middle-east', NULL, 'none', NULL, NULL, 38),
+('MAR', 'ma', '504', 'Morocco', 'middle-east', NULL, NULL, NULL, NULL, 39),
+('SAU', 'sa', '682', 'Saudi Arabia', 'middle-east', NULL, 'none', NULL, NULL, 40),
+('NGA', 'ng', '566', 'Nigeria', 'africa', NULL, NULL, NULL, NULL, 41),
+('ETH', 'et', '231', 'Ethiopia', 'africa', NULL, NULL, NULL, NULL, 42),
+('KEN', 'ke', '404', 'Kenya', 'africa', NULL, NULL, NULL, NULL, 43),
+('ZAF', 'za', '710', 'South Africa', 'africa', NULL, 'history', 'Politics still shaped by apartheid and the liberation struggle', NULL, 44),
+('IND', 'in', '356', 'India', 'south-asia', NULL, 'culture', 'Main divide: Hindu nationalism vs secular pluralism', '{"center": "Largest democracy; religion, caste and language drive politics", "fights": ["Hindu nationalism", "caste quotas", "language", "center vs states"], "timeline": [{"year": "1858", "label": "British Raj"}, {"year": "1947", "label": "independence and Partition"}, {"year": "1961", "label": "Non-Aligned Movement"}, {"year": "1991", "label": "economic liberalization"}, {"year": "2014", "label": "BJP majority"}], "blindSpots": ["polarized media", "reduced to poverty & religion by outsiders"]}'::jsonb, 45),
+('PAK', 'pk', '586', 'Pakistan', 'south-asia', NULL, NULL, NULL, NULL, 46),
+('BGD', 'bd', '050', 'Bangladesh', 'south-asia', NULL, 'history', NULL, NULL, 47),
+('CHN', 'cn', '156', 'China', 'east-asia', 'western', 'none', '"Liberal" = pro-Western intellectuals, a label the state treats with suspicion', '{"center": "Order, growth and national rejuvenation, led by the Party", "fights": ["youth jobs", "property crisis", "local corruption"], "timeline": [{"year": "1839", "label": "Opium Wars, \"humiliation\""}, {"year": "1937", "label": "Japanese invasion"}, {"year": "1966", "label": "Cultural Revolution"}, {"year": "1978", "label": "reform and opening"}, {"year": "1989", "label": "Tiananmen (unspoken)"}], "blindSpots": ["censorship shapes history", "criticism read as new humiliation"]}'::jsonb, 48),
+('JPN', 'jp', '392', 'Japan', 'east-asia', 'market', 'history', 'The "Liberal Democratic Party" is the conservative party in power since 1955', '{"center": "Stability and harmony; one party governing since 1955", "fights": ["pacifist constitution", "aging & immigration", "war memory"], "timeline": [{"year": "1868", "label": "Meiji modernization"}, {"year": "1937", "label": "war in Asia"}, {"year": "1945", "label": "atomic bombs, defeat"}, {"year": "1947", "label": "pacifist constitution"}, {"year": "1990", "label": "bubble bursts"}], "blindSpots": ["wartime atrocities taught thinly", "sees itself as war victim"]}'::jsonb, 49),
+('KOR', 'kr', '410', 'South Korea', 'east-asia', 'left', 'history', 'Divide is mostly about North Korea, Japan and the US', '{"center": "Fierce democracy won by protest; hyper-competitive society", "fights": ["North Korea", "Japan", "gender", "chaebol power"], "timeline": [{"year": "1910", "label": "Japanese colonization"}, {"year": "1950", "label": "Korean War"}, {"year": "1980", "label": "Gwangju massacre"}, {"year": "1987", "label": "democracy"}, {"year": "2024", "label": "martial law crisis"}], "blindSpots": ["US and Japan seen through party lens"]}'::jsonb, 50),
+('TWN', 'tw', '158', 'Taiwan', 'east-asia', NULL, 'history', 'Main divide: how close to be with mainland China', NULL, 51),
+('VNM', 'vn', '704', 'Vietnam', 'southeast-asia', NULL, 'none', NULL, NULL, 52),
+('THA', 'th', '764', 'Thailand', 'southeast-asia', NULL, 'history', 'Main divide: monarchy and military vs reformists', NULL, 53),
+('MYS', 'my', '458', 'Malaysia', 'southeast-asia', NULL, 'culture', NULL, NULL, 54),
+('PHL', 'ph', '608', 'Philippines', 'southeast-asia', NULL, NULL, NULL, NULL, 55),
+('IDN', 'id', '360', 'Indonesia', 'southeast-asia', NULL, 'culture', NULL, NULL, 56);
+--> statement-breakpoint
+INSERT INTO "stats"."wo_cultural_map" ("iso3","wave","trad_secular","surv_selfexp","source") VALUES
+('SWE', 'curated', 1.8, 2.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('DNK', 'curated', 1.2, 2.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('NOR', 'curated', 1.4, 2.1, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('FIN', 'curated', 1, 1.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('NLD', 'curated', 1.1, 2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('BEL', 'curated', 0.8, 1.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('DEU', 'curated', 1.4, 1.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('AUT', 'curated', 0.6, 1.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('CHE', 'curated', 0.7, 1.9, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('GBR', 'curated', 0.3, 1.8, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('IRL', 'curated', -0.4, 1.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('FRA', 'curated', 0.8, 1.1, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('ESP', 'curated', 0.3, 1.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('PRT', 'curated', -0.3, 0.8, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('ITA', 'curated', 0, 0.5, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('GRC', 'curated', 0, 0.2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('USA', 'curated', -0.5, 1.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('CAN', 'curated', 0, 1.8, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('MEX', 'curated', -1.3, 0.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('BRA', 'curated', -1, 0.1, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('ARG', 'curated', -0.6, 0.7, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('CHL', 'curated', -0.3, 0.8, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('COL', 'curated', -1.6, 0.5, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('PER', 'curated', -1.1, 0.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('VEN', 'curated', -1.5, 0.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('AUS', 'curated', 0.2, 2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('NZL', 'curated', 0.1, 2.2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('POL', 'curated', -0.6, 0.2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('CZE', 'curated', 1.2, 0.5, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('HUN', 'curated', 0.4, -0.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('ROU', 'curated', -0.6, -0.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('SRB', 'curated', 0.1, -0.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('UKR', 'curated', 0.2, -0.9, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('RUS', 'curated', 0.6, -1, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('KAZ', 'curated', 0.4, -0.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('TUR', 'curated', -0.9, -0.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('IRN', 'curated', -0.6, -0.9, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('IRQ', 'curated', -1.3, -1.2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('EGY', 'curated', -1.5, -1.5, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('MAR', 'curated', -1.5, -1.2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('NGA', 'curated', -1.8, -0.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('ETH', 'curated', -1.5, -0.9, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('KEN', 'curated', -1.5, -0.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('ZAF', 'curated', -1.2, 0, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('IND', 'curated', -0.5, -0.5, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('PAK', 'curated', -1.4, -1, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('BGD', 'curated', -1.1, -1, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('CHN', 'curated', 1.5, -0.9, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('JPN', 'curated', 1.8, 0.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('KOR', 'curated', 1.2, -0.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('TWN', 'curated', 1.3, 0, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('VNM', 'curated', 0.5, -0.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('THA', 'curated', -0.4, 0, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('MYS', 'curated', -0.9, -0.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('PHL', 'curated', -1.4, -0.2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
+('IDN', 'curated', -1.2, -0.8, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map');
+--> statement-breakpoint
+INSERT INTO "stats"."wo_narrative" ("sort","event","years","views") VALUES
+(0, 'World War II', '1939–45', '[{"country": "us", "story": "D-Day: democracy beats fascism"}, {"country": "ru", "story": "We broke Hitler, 27M dead"}, {"country": "cn", "story": "Our war began in 1937"}, {"country": "pl", "story": "Liberation became Soviet occupation"}, {"country": "jp", "story": "Hiroshima: we suffered"}]'::jsonb),
+(1, 'End of the Cold War', '1989–91', '[{"country": "us", "story": "We won: the end of history"}, {"country": "pl", "story": "We freed ourselves"}, {"country": "ru", "story": "Collapse and humiliation"}, {"country": "cn", "story": "A warning: never lose control"}]'::jsonb),
+(2, 'Colonialism', '1500s–1970s', '[{"country": "nl", "story": "Golden Age trade, now debated"}, {"country": "gb", "story": "Empire, remembered fondly by many"}, {"country": "in", "story": "Extraction and Partition"}, {"country": "cn", "story": "Century of humiliation"}, {"country": "br", "story": "Conquest, then US dominance"}]'::jsonb),
+(3, 'Iraq War', '2003', '[{"country": "us", "story": "Disarmament, later a mistake"}, {"country": "fr", "story": "We warned you"}, {"country": "iq", "story": "Invasion, then civil war"}, {"country": "ru", "story": "Proof the rules are selective"}]'::jsonb);
+--> statement-breakpoint
+INSERT INTO "stats"."wo_source" ("sort","name","href") VALUES
+(0, 'World Values Survey / EVS cultural map', 'https://www.worldvaluessurvey.org/WVSContents.jsp?CMSID=Findings'),
+(1, 'Chapel Hill Expert Survey', 'https://www.chesdata.eu/'),
+(2, 'Manifesto Project', 'https://manifesto-project.wzb.eu/'),
+(3, 'Pew Research Center', 'https://www.pewresearch.org/global/'),
+(4, 'Natural Earth (map shapes)', 'https://www.naturalearthdata.com/');
+--> statement-breakpoint
+INSERT INTO "stats"."wo_party" ("party_id","iso3","name","abbrev","sort") VALUES
+('wo:us:Dem', 'USA', 'Democratic Party', 'Dem', 0),
+('wo:us:GOP', 'USA', 'Republican Party', 'GOP', 1),
+('wo:nl:GL-PvdA', 'NLD', 'GroenLinks-PvdA', 'GL-PvdA', 2),
+('wo:nl:SP', 'NLD', 'Socialistische Partij', 'SP', 3),
+('wo:nl:D66', 'NLD', 'Democraten 66', 'D66', 4),
+('wo:nl:CDA', 'NLD', 'Christen-Democratisch Appèl', 'CDA', 5),
+('wo:nl:VVD', 'NLD', 'Volkspartij voor Vrijheid en Democratie', 'VVD', 6),
+('wo:nl:BBB', 'NLD', 'BoerBurgerBeweging', 'BBB', 7),
+('wo:nl:PVV', 'NLD', 'Partij voor de Vrijheid', 'PVV', 8),
+('wo:de:Linke', 'DEU', 'Die Linke', 'Linke', 9),
+('wo:de:Grüne', 'DEU', 'Bündnis 90/Die Grünen', 'Grüne', 10),
+('wo:de:SPD', 'DEU', 'Sozialdemokratische Partei', 'SPD', 11),
+('wo:de:BSW', 'DEU', 'Bündnis Sahra Wagenknecht', 'BSW', 12),
+('wo:de:FDP', 'DEU', 'Freie Demokratische Partei', 'FDP', 13),
+('wo:de:CDU', 'DEU', 'CDU/CSU', 'CDU', 14),
+('wo:de:AfD', 'DEU', 'Alternative für Deutschland', 'AfD', 15),
+('wo:gb:Green', 'GBR', 'Green Party', 'Green', 16),
+('wo:gb:Lab', 'GBR', 'Labour', 'Lab', 17),
+('wo:gb:LD', 'GBR', 'Liberal Democrats', 'LD', 18),
+('wo:gb:Con', 'GBR', 'Conservatives', 'Con', 19),
+('wo:gb:Reform', 'GBR', 'Reform UK', 'Reform', 20),
+('wo:fr:LFI', 'FRA', 'La France Insoumise', 'LFI', 21),
+('wo:fr:PS', 'FRA', 'Parti Socialiste', 'PS', 22),
+('wo:fr:RE', 'FRA', 'Renaissance', 'RE', 23),
+('wo:fr:LR', 'FRA', 'Les Républicains', 'LR', 24),
+('wo:fr:RN', 'FRA', 'Rassemblement National', 'RN', 25),
+('wo:se:V', 'SWE', 'Vänsterpartiet', 'V', 26),
+('wo:se:S', 'SWE', 'Socialdemokraterna', 'S', 27),
+('wo:se:C', 'SWE', 'Centerpartiet', 'C', 28),
+('wo:se:M', 'SWE', 'Moderaterna', 'M', 29),
+('wo:se:SD', 'SWE', 'Sverigedemokraterna', 'SD', 30),
+('wo:pl:Lewica', 'POL', 'Lewica', 'Lewica', 31),
+('wo:pl:KO', 'POL', 'Koalicja Obywatelska', 'KO', 32),
+('wo:pl:PiS', 'POL', 'Prawo i Sprawiedliwość', 'PiS', 33),
+('wo:pl:Konf', 'POL', 'Konfederacja', 'Konf', 34),
+('wo:jp:JCP', 'JPN', 'Japanese Communist Party', 'JCP', 35),
+('wo:jp:CDP', 'JPN', 'Constitutional Democratic Party', 'CDP', 36),
+('wo:jp:Komeito', 'JPN', 'Komeito', 'Komeito', 37),
+('wo:jp:LDP', 'JPN', 'Liberal Democratic Party', 'LDP', 38),
+('wo:jp:Ishin', 'JPN', 'Nippon Ishin no Kai', 'Ishin', 39),
+('wo:jp:Sansei', 'JPN', 'Sanseitō', 'Sansei', 40),
+('wo:kr:Justice', 'KOR', 'Justice Party', 'Justice', 41),
+('wo:kr:DPK', 'KOR', 'Democratic Party of Korea', 'DPK', 42),
+('wo:kr:PPP', 'KOR', 'People Power Party', 'PPP', 43),
+('wo:in:INC', 'IND', 'Indian National Congress', 'INC', 44),
+('wo:in:BJP', 'IND', 'Bharatiya Janata Party', 'BJP', 45),
+('wo:br:PT', 'BRA', 'Partido dos Trabalhadores', 'PT', 46),
+('wo:br:MDB', 'BRA', 'Movimento Democrático Brasileiro', 'MDB', 47),
+('wo:br:PL', 'BRA', 'Partido Liberal', 'PL', 48),
+('wo:ca:NDP', 'CAN', 'New Democratic Party', 'NDP', 49),
+('wo:ca:LPC', 'CAN', 'Liberal Party of Canada', 'LPC', 50),
+('wo:ca:CPC', 'CAN', 'Conservative Party of Canada', 'CPC', 51);
+--> statement-breakpoint
+INSERT INTO "stats"."wo_compass_override" ("party_id","x","y","note") VALUES
+('wo:us:Dem', 4.8, 3.5, 'The US "liberals": near Europe''s center economically'),
+('wo:us:GOP', 7.8, 8, NULL),
+('wo:nl:GL-PvdA', 2.5, 1.5, NULL),
+('wo:nl:SP', 1.5, 5, NULL),
+('wo:nl:D66', 5.5, 1.5, NULL),
+('wo:nl:CDA', 6, 6, NULL),
+('wo:nl:VVD', 7.5, 5, 'Dutch "liberals": market-liberal, center-right'),
+('wo:nl:BBB', 6, 7.5, NULL),
+('wo:nl:PVV', 4.5, 9, 'Radical right on immigration, centrist on welfare'),
+('wo:de:Linke', 1, 2.5, NULL),
+('wo:de:Grüne', 3, 1.5, NULL),
+('wo:de:SPD', 3.5, 3.5, NULL),
+('wo:de:BSW', 2, 7, NULL),
+('wo:de:FDP', 8.5, 3, 'German "liberals": socially open, strongly pro-market'),
+('wo:de:CDU', 6.5, 6, NULL),
+('wo:de:AfD', 7, 9.5, NULL),
+('wo:gb:Green', 1.5, 1.5, NULL),
+('wo:gb:Lab', 3.8, 3.5, NULL),
+('wo:gb:LD', 5, 2.5, NULL),
+('wo:gb:Con', 7, 6.5, NULL),
+('wo:gb:Reform', 7.5, 8.5, NULL),
+('wo:fr:LFI', 1, 2.5, NULL),
+('wo:fr:PS', 3, 3, NULL),
+('wo:fr:RE', 6.5, 4, NULL),
+('wo:fr:LR', 7.5, 7, NULL),
+('wo:fr:RN', 4.5, 9, 'Economically left of Macron: lower retirement age, protectionism'),
+('wo:se:V', 1, 2, NULL),
+('wo:se:S', 3, 3.5, NULL),
+('wo:se:C', 7, 2.5, NULL),
+('wo:se:M', 7.5, 5.5, NULL),
+('wo:se:SD', 5.5, 9, NULL),
+('wo:pl:Lewica', 2, 2, NULL),
+('wo:pl:KO', 6.5, 4, NULL),
+('wo:pl:PiS', 3.5, 9, 'Culturally right, economically left (big family benefits)'),
+('wo:pl:Konf', 9, 9, NULL),
+('wo:jp:JCP', 1, 3, NULL),
+('wo:jp:CDP', 4, 3.5, NULL),
+('wo:jp:Komeito', 5, 5.5, NULL),
+('wo:jp:LDP', 6, 7, 'Called "Liberal", governs as conservatives since 1955'),
+('wo:jp:Ishin', 7.5, 6, NULL),
+('wo:jp:Sansei', 6, 9, NULL),
+('wo:kr:Justice', 1.5, 1.5, NULL),
+('wo:kr:DPK', 4, 4.5, 'The "progressive" camp, cautious on social issues'),
+('wo:kr:PPP', 7, 7, NULL),
+('wo:in:INC', 4.5, 4, NULL),
+('wo:in:BJP', 6, 8.5, 'Hindu-nationalist, pro-business, big welfare schemes'),
+('wo:br:PT', 2.5, 4.5, NULL),
+('wo:br:MDB', 6, 6, NULL),
+('wo:br:PL', 7.5, 9, NULL),
+('wo:ca:NDP', 2.5, 2.5, NULL),
+('wo:ca:LPC', 4.8, 3, NULL),
+('wo:ca:CPC', 7.5, 7, NULL);

scripts/deploy-preflight.ts

@@ -177,7 +177,7 @@ try {
 		throw new Error(`groceries schema grants are wrong: ${JSON.stringify(groceries)}`);
 	// Public /stats section: open data, so the app role (and the host-side stats collector, which
 	// uses it) reads and writes, and personal_ro may read. Bump the count with every new stats table.
-	const STATS_TABLES = 32;
+	const STATS_TABLES = 49;
 	const [stats] = await client`
 		select
 			count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,

scripts/stats-fetch.ts

@@ -12,6 +12,9 @@
 //   bun scripts/stats-fetch.ts eponline [--force]   (needs EP_ONLINE_API_KEY)
 //   bun scripts/stats-fetch.ts cbscw [--force] [--only=cpi,gdp,…]   CBS Curaçao files
 //   bun scripts/stats-fetch.ts cwintl [--only=ilostat,owid,unsdg,comtrade]   Curaçao via international APIs
+//   bun scripts/stats-fetch.ts elections [--force] [--only=kiesraad,cbs,mit]   A14
+//   bun scripts/stats-fetch.ts parties            CHES + V-Party + Party Facts (A15)
+//   bun scripts/stats-fetch.ts wikipolls|eurobarometer [--force]|whr|gtrends|gdelt   A16
 //   bun scripts/stats-fetch.ts normalise-cbs     rebuild CBS chart series from the raw rows
 import { ensureMeta, sql } from './stats/common';
 import { cbs, normaliseCbs } from './stats/cbs';
@@ -28,6 +31,13 @@ import { usaspending } from './stats/usaspending';
 import { eponline } from './stats/eponline';
 import { cwintl } from './stats/cwintl';
 import { cwcbs } from './stats/cwcbs';
+import { elections } from './stats/elections';
+import { parties } from './stats/parties';
+import { wikipolls } from './stats/wikipolls';
+import { eurobarometer } from './stats/eurobarometer';
+import { whr } from './stats/whr';
+import { gtrends } from './stats/gtrends';
+import { gdelt } from './stats/gdelt';
 
 const [source, ...args] = process.argv.slice(2);
 const flag = (name: string) => args.includes(`--${name}`);
@@ -82,9 +92,30 @@ try {
 		case 'eponline':
 			await eponline({ force: flag('force') || flag('backfill'), job });
 			break;
+		case 'elections':
+			await elections({ force: flag('force') || flag('backfill'), only: value('only'), job });
+			break;
+		case 'parties':
+			await parties({ job });
+			break;
+		case 'wikipolls':
+			await wikipolls({ job });
+			break;
+		case 'eurobarometer':
+			await eurobarometer({ force: flag('force') || flag('backfill'), job });
+			break;
+		case 'whr':
+			await whr({ job });
+			break;
+		case 'gtrends':
+			await gtrends({ job });
+			break;
+		case 'gdelt':
+			await gdelt({ job, timespan: value('timespan') });
+			break;
 		default:
 			throw new Error(
-				`Unknown source "${source}" (cbs, eurostat, fred, bis, worldbank, imf, owid, energycharts, rijksfinancien, ustreasury, usaspending, eponline, cbscw, cwintl, normalise-cbs)`
+				`Unknown source "${source}" (cbs, eurostat, fred, bis, worldbank, imf, owid, energycharts, rijksfinancien, ustreasury, usaspending, eponline, cbscw, cwintl, elections, parties, wikipolls, eurobarometer, whr, gtrends, gdelt, normalise-cbs)`
 			);
 	}
 } finally {

scripts/stats/common.ts

@@ -37,7 +37,17 @@ const hostGapMs: Record<string, number> = {
 	'cuatro.sim-cdn.nl': 1000,
 	'sdmx.ilo.org': 3000,
 	'unstats.un.org': 1500,
-	'comtradeapi.un.org': 2500
+	'comtradeapi.un.org': 2500,
+	'data.overheid.nl': 500,
+	'dataverse.harvard.edu': 1000,
+	'github.com': 1000,
+	'partyfacts.herokuapp.com': 2000,
+	'en.wikipedia.org': 1000,
+	'data.europa.eu': 1000,
+	'webgate.ec.europa.eu': 2000,
+	'files.worldhappiness.report': 1000,
+	'trends.google.com': 3000,
+	'api.gdeltproject.org': 5500
 };
 const lastHit = new Map<string, number>();
 
@@ -54,7 +64,7 @@ export async function http(url: string, init: RequestInit = {}, tries = 4): Prom
 			response = await fetch(url, {
 				...init,
 				headers: { 'user-agent': USER_AGENT, accept: '*/*', ...(init.headers ?? {}) },
-				signal: AbortSignal.timeout(180_000)
+				signal: init.signal ?? AbortSignal.timeout(180_000)
 			});
 			if (response.ok) return response;
 			if (response.status !== 429 && response.status < 500) {
@@ -248,6 +258,74 @@ export const SOURCES = [
 		licence: 'Free use with attribution (UN Comtrade public preview)',
 		key_required: false,
 		attribution: 'Source: UN Comtrade Database (United Nations Statistics Division)'
+	},
+	{
+		source_id: 'kiesraad',
+		name: 'Kiesraad election results (data.overheid.nl)',
+		base_url: 'https://data.overheid.nl/dataset?q=verkiezingsuitslag',
+		licence: 'CC0 1.0',
+		key_required: false,
+		attribution: 'Source: Kiesraad, verkiezingsuitslagen (data.overheid.nl), CC0'
+	},
+	{
+		source_id: 'mitelection',
+		name: 'MIT Election Data and Science Lab, U.S. President 1976–2024',
+		base_url: 'https://doi.org/10.7910/DVN/42MVDX',
+		licence: 'CC0 1.0',
+		key_required: false,
+		attribution:
+			'Source: MIT Election Data and Science Lab, U.S. President 1976–2024 (Harvard Dataverse), CC0'
+	},
+	{
+		source_id: 'ches',
+		name: 'Chapel Hill Expert Survey (trend file) + V-Party + Party Facts',
+		base_url: 'https://www.chesdata.eu',
+		licence:
+			'Free for non-commercial research use with citation (CHES); V-Dem CC BY-SA 4.0; Party Facts CC BY-SA 4.0',
+		key_required: false,
+		attribution:
+			'Sources: Chapel Hill Expert Survey 1999–2024 (Jolly et al.); V-Dem V-Party v2 (Lührmann et al.); Party Facts (Döring & Regel)'
+	},
+	{
+		source_id: 'wikipedia',
+		name: 'Wikipedia opinion-polling tables',
+		base_url: 'https://en.wikipedia.org',
+		licence: 'CC BY-SA 4.0',
+		key_required: false,
+		attribution:
+			'Source: Wikipedia opinion-polling articles and their cited pollsters, CC BY-SA 4.0'
+	},
+	{
+		source_id: 'eurobarometer',
+		name: 'Standard Eurobarometer (volume A tables)',
+		base_url: 'https://data.europa.eu',
+		licence: 'Reuse authorised with acknowledgement (Commission Decision 2011/833/EU)',
+		key_required: false,
+		attribution: 'Source: European Commission, Standard Eurobarometer, © European Union'
+	},
+	{
+		source_id: 'whr',
+		name: 'World Happiness Report (Figure 2.1 data)',
+		base_url: 'https://worldhappiness.report',
+		licence: 'Free reuse with attribution',
+		key_required: false,
+		attribution: 'Source: World Happiness Report (Wellbeing Research Centre, Gallup World Poll)'
+	},
+	{
+		source_id: 'gtrends',
+		name: 'Google Trends, trending now (RSS)',
+		base_url: 'https://trends.google.com/trending/rss',
+		licence: 'Google Terms of Service; shown with attribution',
+		key_required: false,
+		attribution: 'Source: Google Trends'
+	},
+	{
+		source_id: 'gdelt',
+		name: 'GDELT Project DOC 2.0 API',
+		base_url: 'https://api.gdeltproject.org/api/v2/doc/doc',
+		licence: 'Open, unrestricted use with citation',
+		key_required: false,
+		attribution: 'Source: The GDELT Project'
 	}
 ] as const;
 

scripts/stats/elections.ts

@@ -0,0 +1,543 @@
+// A14 elections: NL Tweede Kamer from the Kiesraad open data (data.overheid.nl, CC0) and CBS
+// 37278 (1918–2006), US presidential results by state from the MIT Election Lab (CC0).
+//  - Kiesraad: per election the "CSV formaat" zip (votes per list per municipality and per
+//    polling station; 2023+ also turnout and seats) plus the EML zips, read with HTTP ranges, for
+//    the official result (seats, election date) and, before 2023, the national total count.
+//    Polling stations are kept for Gouda (GM0513) only.
+//  - Results are final, so an election already loaded is skipped unless --force.
+import { insertChunks, getJson, http, log, run, sql } from './common';
+import { parseCsv } from './owid';
+import { RemoteZip } from './remotezip';
+
+const CKAN =
+	'https://data.overheid.nl/data/api/3/action/package_search?q=verkiezingsuitslag%20Tweede%20Kamer&rows=100';
+const STATION_MUNIS = new Set(['GM0513']);
+
+type Result = {
+	election_id: string;
+	level: string;
+	area_code: string;
+	area_name: string | null;
+	party: string;
+	votes: number | null;
+	seats: number | null;
+};
+type Turnout = {
+	election_id: string;
+	level: string;
+	area_code: string;
+	area_name: string | null;
+	electorate: number | null;
+	ballots: number | null;
+	valid: number | null;
+	blank: number | null;
+	invalid: number | null;
+};
+type Election = {
+	election_id: string;
+	country: string;
+	kind: string;
+	date: string;
+	name: string;
+	source_id: string;
+	total_seats: number | null;
+};
+
+async function save(e: Election, results: Result[], turnout: Turnout[]) {
+	// Merge duplicate keys (a list name can repeat within a municipality in old files).
+	const merged = new Map<string, Result>();
+	for (const r of results) {
+		const key = `${r.level}|${r.area_code}|${r.party}`;
+		const prev = merged.get(key);
+		if (!prev) merged.set(key, { ...r });
+		else {
+			prev.votes =
+				prev.votes == null && r.votes == null ? null : (prev.votes ?? 0) + (r.votes ?? 0);
+			prev.seats = prev.seats ?? r.seats;
+		}
+	}
+	await sql.begin(async (tx) => {
+		await tx`delete from stats.elec_result where election_id = ${e.election_id}`;
+		await tx`delete from stats.elec_turnout where election_id = ${e.election_id}`;
+		await tx`delete from stats.elec_election where election_id = ${e.election_id}`;
+		await insertChunks(tx, 'elec_election', [e]);
+		await insertChunks(tx, 'elec_result', [...merged.values()]);
+		await insertChunks(tx, 'elec_turnout', turnout);
+	});
+	return merged.size + turnout.length;
+}
+
+const num = (s: string | undefined) => {
+	if (s == null || s.trim() === '') return null;
+	const n = Number(s.replace(/\s/g, ''));
+	return Number.isFinite(n) ? n : null;
+};
+const gm = (code: string) => `GM${code.replace(/^\D+/, '').padStart(4, '0')}`;
+const xmlText = (s: string) =>
+	s
+		.replace(/&lt;/g, '<')
+		.replace(/&gt;/g, '>')
+		.replace(/&quot;/g, '"')
+		.replace(/&apos;/g, "'")
+		.replace(/&amp;/g, '&');
+
+/** Rows of a semicolon CSV as objects keyed by the header. */
+function rowsOf(text: string): Record<string, string>[] {
+	const [header, ...rows] = parseCsv(text, ';');
+	return rows
+		.filter((r) => r.length >= header.length - 1 && r.some((x) => x !== ''))
+		.map((r) => Object.fromEntries(header.map((h, i) => [h.trim(), (r[i] ?? '').trim()])));
+}
+
+/** EML 520 (Resultaat): election date and seats per list (elected candidates per affiliation). */
+function parseResultaat(xml: string) {
+	const date = xml.match(/<kr:ElectionDate>([\d-]+)<\/kr:ElectionDate>/)?.[1] ?? null;
+	const seats = new Map<string, number>();
+	let current = '';
+	for (const m of xml.matchAll(/<Selection>([\s\S]*?)<\/Selection>/g)) {
+		const aff = m[1].match(/<RegisteredName>([\s\S]*?)<\/RegisteredName>/);
+		if (aff) {
+			current = xmlText(aff[1].trim());
+			if (!seats.has(current)) seats.set(current, 0);
+		} else if (/<Candidate>/.test(m[1]) && /<Elected>yes<\/Elected>/.test(m[1]))
+			seats.set(current, (seats.get(current) ?? 0) + 1);
+	}
+	return { date, seats };
+}
+
+/** EML 510d (Totaaltelling): national votes per list and the count totals. */
+function parseTotaaltelling(xml: string) {
+	const total = xml.match(/<TotalVotes>([\s\S]*?)<\/TotalVotes>/)?.[1] ?? '';
+	const votes = new Map<string, number>();
+	for (const m of total.matchAll(
+		/<Selection><AffiliationIdentifier[^>]*><RegisteredName>([\s\S]*?)<\/RegisteredName><\/AffiliationIdentifier><ValidVotes>(\d+)<\/ValidVotes>/g
+	))
+		votes.set(xmlText(m[1].trim()), Number(m[2]));
+	const pick = (re: RegExp) => num(total.match(re)?.[1]);
+	return {
+		votes,
+		electorate: pick(/<Cast>(\d+)<\/Cast>/),
+		ballots: pick(/<TotalCounted>(\d+)<\/TotalCounted>/),
+		invalid: pick(/<RejectedVotes ReasonCode="ongeldig">(\d+)</),
+		blank: pick(/<RejectedVotes ReasonCode="blanco">(\d+)</)
+	};
+}
+
+type CkanResource = { url: string; name?: string; format?: string };
+type CkanPackage = { name: string; title: string; resources: CkanResource[] };
+
+async function kiesraad(force: boolean, note: (w: string) => void) {
+	const res = await getJson<{ result: { results: CkanPackage[] } }>(CKAN);
+	const packages = res.result.results
+		.map((p) => ({ p, year: p.name.match(/^verkiezingsuitslag-tweede-kamer-(\d{4})$/)?.[1] }))
+		.filter((x): x is { p: CkanPackage; year: string } => !!x.year)
+		.sort((a, b) => a.year.localeCompare(b.year));
+	const loaded = new Set(
+		(await sql`select election_id from stats.elec_election where source_id = 'kiesraad'`).map(
+			(r) => r.election_id as string
+		)
+	);
+	let rows = 0;
+	for (const { p, year } of packages) {
+		const id = `NL-TK-${year}`;
+		if (loaded.has(id) && !force) continue;
+		const csvRes = p.resources.find((r) => /csv/i.test(decodeURIComponent(r.url)));
+		if (!csvRes) {
+			note(`${id}: no CSV zip`);
+			continue;
+		}
+		const csvZip = await RemoteZip.open(csvRes.url);
+		// EML zips: find the official result (and, for older years, the national count).
+		let resultaat: ReturnType<typeof parseResultaat> | null = null;
+		let totaal: ReturnType<typeof parseTotaaltelling> | null = null;
+		for (const r of p.resources.filter((r) => r !== csvRes && /\.zip$/i.test(r.url))) {
+			const zip = await RemoteZip.open(r.url);
+			const re = zip.find(/(^|\/)Resultaat_TK\d+[^/]*\.eml\.xml$/i)[0];
+			if (re && !resultaat) resultaat = parseResultaat(await zip.text(re));
+			const tt = zip.find(/(^|\/)Totaaltelling_TK\d+[^/]*\.eml\.xml$/i)[0];
+			if (tt && !totaal && !csvZip.find(/_uitslag\.csv$/i).length)
+				totaal = parseTotaaltelling(await zip.text(tt));
+			if (resultaat && (totaal || csvZip.find(/_uitslag\.csv$/i).length)) break;
+		}
+		const results: Result[] = [];
+		const turnout: Turnout[] = [];
+		const base = { election_id: id };
+		const uitslag = csvZip.find(/_uitslag\.csv$/i)[0];
+		if (uitslag) {
+			for (const r of rowsOf(await csvZip.text(uitslag))) {
+				const code = r.RegioCode ?? '';
+				const level = code.startsWith('L') ? 'national' : /^[GO]/.test(code) ? 'muni' : null;
+				if (!level) continue;
+				const area_code = level === 'national' ? 'NL' : gm(code);
+				const area_name = level === 'national' ? 'Nederland' : r.Regio;
+				const v = num(r.Waarde);
+				if (r.VeldType === 'LijstAantalStemmen' && r.LijstNaam)
+					results.push({
+						...base,
+						level,
+						area_code,
+						area_name,
+						party: r.LijstNaam,
+						votes: v,
+						seats: null
+					});
+				else if (r.VeldType === 'LijstAantalZetels' && r.LijstNaam && level === 'national')
+					results.push({
+						...base,
+						level,
+						area_code,
+						area_name,
+						party: r.LijstNaam,
+						votes: null,
+						seats: v
+					});
+				else if (
+					[
+						'Kiesgerechtigden',
+						'Opkomst',
+						'AantalGeldigeStemmen',
+						'AantalBlancoStemmen',
+						'AantalOngeldigeStemmen'
+					].includes(r.VeldType)
+				) {
+					let t = turnout.find((x) => x.level === level && x.area_code === area_code);
+					if (!t) {
+						t = {
+							...base,
+							level,
+							area_code,
+							area_name,
+							electorate: null,
+							ballots: null,
+							valid: null,
+							blank: null,
+							invalid: null
+						};
+						turnout.push(t);
+					}
+					const col = {
+						Kiesgerechtigden: 'electorate',
+						Opkomst: 'ballots',
+						AantalGeldigeStemmen: 'valid',
+						AantalBlancoStemmen: 'blank',
+						AantalOngeldigeStemmen: 'invalid'
+					}[r.VeldType] as 'electorate' | 'ballots' | 'valid' | 'blank' | 'invalid';
+					t[col] = v;
+				}
+			}
+		} else {
+			const gem = csvZip.find(/Per_Gemeente\.csv$/i)[0];
+			if (!gem) throw new Error(`${id}: no municipality CSV in ${csvRes.url}`);
+			for (const r of rowsOf(await csvZip.text(gem)))
+				results.push({
+					...base,
+					level: 'muni',
+					area_code: gm(r.GemeenteCode),
+					area_name: r.GemeenteNaam,
+					party: r.PartijNaam,
+					votes: num(r.AantalStemmen),
+					seats: null
+				});
+			if (totaal) {
+				const seats = resultaat?.seats ?? new Map<string, number>();
+				for (const [party, votes] of totaal.votes)
+					results.push({
+						...base,
+						level: 'national',
+						area_code: 'NL',
+						area_name: 'Nederland',
+						party,
+						votes,
+						seats: seats.get(party) ?? null
+					});
+				const valid = [...totaal.votes.values()].reduce((a, b) => a + b, 0);
+				turnout.push({
+					...base,
+					level: 'national',
+					area_code: 'NL',
+					area_name: 'Nederland',
+					electorate: totaal.electorate,
+					ballots: totaal.ballots,
+					valid,
+					blank: totaal.blank,
+					invalid: totaal.invalid
+				});
+			} else note(`${id}: no Totaaltelling EML, national total missing`);
+		}
+		// Merge votes and seats rows of the national level (2023+ lists them separately).
+		const nat = new Map<string, Result>();
+		for (const r of results.filter((r) => r.level === 'national')) {
+			const prev = nat.get(r.party);
+			if (prev) {
+				prev.votes ??= r.votes;
+				prev.seats ??= r.seats;
+			} else nat.set(r.party, r);
+		}
+		const finalResults = [...results.filter((r) => r.level !== 'national'), ...nat.values()];
+		// Polling stations (Gouda only).
+		const sb = csvZip.find(/Per_Stembureau\.csv$/i)[0];
+		if (sb) {
+			for (const r of rowsOf(await csvZip.text(sb))) {
+				const muni = gm(r.GemeenteCode);
+				if (!STATION_MUNIS.has(muni)) continue;
+				finalResults.push({
+					...base,
+					level: 'station',
+					area_code: `${muni}-${r.StembureauCode}`,
+					area_name: r.StembureauNaam || null,
+					party: r.PartijNaam,
+					votes: num(r.AantalStemmen),
+					seats: null
+				});
+			}
+		} else note(`${id}: no polling-station CSV`);
+		const totalSeats = [...nat.values()].reduce((a, r) => a + (r.seats ?? 0), 0) || null;
+		const date = resultaat?.date ?? KNOWN_DATES[year];
+		if (!date) throw new Error(`${id}: no election date`);
+		if (!resultaat) note(`${id}: no Resultaat EML (seats/date from CSV or table)`);
+		const n = await save(
+			{
+				election_id: id,
+				country: 'NLD',
+				kind: 'TK',
+				date,
+				name: `Tweede Kamer ${year}`,
+				source_id: 'kiesraad',
+				total_seats: totalSeats
+			},
+			finalResults,
+			turnout
+		);
+		log(
+			`kiesraad ${id}: ${n} rows (${finalResults.filter((r) => r.level === 'station').length} Gouda station rows)`
+		);
+		rows += n;
+	}
+	return rows;
+}
+
+// Election days (CBS 37278 only has the year). Source: Kiesraad / Parlement.com.
+const KNOWN_DATES: Record<string, string> = {
+	'1918': '1918-07-03',
+	'1922': '1922-07-05',
+	'1925': '1925-07-01',
+	'1929': '1929-07-03',
+	'1933': '1933-04-26',
+	'1937': '1937-05-26',
+	'1946': '1946-05-17',
+	'1948': '1948-07-07',
+	'1952': '1952-06-25',
+	'1956': '1956-06-13',
+	'1959': '1959-03-12',
+	'1963': '1963-05-15',
+	'1967': '1967-02-15',
+	'1971': '1971-04-28',
+	'1972': '1972-11-29',
+	'1977': '1977-05-25',
+	'1981': '1981-05-26',
+	'1982': '1982-09-08',
+	'1986': '1986-05-21',
+	'1989': '1989-09-06',
+	'1994': '1994-05-03',
+	'1998': '1998-05-06',
+	'2002': '2002-05-15',
+	'2003': '2003-01-22',
+	'2006': '2006-11-22',
+	'2010': '2010-06-09',
+	'2012': '2012-09-12',
+	'2017': '2017-03-15',
+	'2021': '2021-03-17',
+	'2023': '2023-11-22',
+	'2025': '2025-10-29'
+};
+
+type CbsProp = { Key: string; Title: string; Type: string; ParentID?: number | null; ID: number };
+
+async function cbs37278() {
+	const base = 'https://opendata.cbs.nl/ODataApi/odata/37278';
+	const props = (await getJson<{ value: CbsProp[] }>(`${base}/DataProperties`)).value;
+	const groups = new Map(props.filter((p) => p.Type === 'TopicGroup').map((p) => [p.ID, p.Title]));
+	const data = (
+		await getJson<{ value: Record<string, string | number | null>[] }>(`${base}/TypedDataSet`)
+	).value;
+	let rows = 0;
+	for (const d of data) {
+		const year = String(d.Perioden).slice(0, 4);
+		const id = `NL-TK-${year}`;
+		const results = new Map<string, Result>();
+		const turnout: Turnout = {
+			election_id: id,
+			level: 'national',
+			area_code: 'NL',
+			area_name: 'Nederland',
+			electorate: null,
+			ballots: null,
+			valid: null,
+			blank: null,
+			invalid: null
+		};
+		for (const p of props.filter((p) => p.Type === 'Topic')) {
+			const v = d[p.Key] == null ? null : Number(d[p.Key]);
+			const group = groups.get(p.ParentID ?? -1) ?? '';
+			if (p.Key.startsWith('Kiesgerechtigden')) turnout.electorate = v;
+			else if (p.Key.startsWith('TotaalStemmen')) turnout.ballots = v;
+			else if (p.Key.startsWith('OngeldigeStemmen')) turnout.invalid = v;
+			else if (p.Key.startsWith('TotaalGeldigeStemmen')) turnout.valid = v;
+			else if (/^(Opkomst|TotaalAantalZetels)/.test(p.Key)) continue;
+			else if (v != null && /stemmen per partij/i.test(group)) {
+				const r = results.get(p.Title) ?? {
+					election_id: id,
+					level: 'national',
+					area_code: 'NL',
+					area_name: 'Nederland',
+					party: p.Title,
+					votes: null,
+					seats: null
+				};
+				r.votes = v;
+				results.set(p.Title, r);
+			} else if (v != null && /zetel/i.test(group)) {
+				const r = results.get(p.Title) ?? {
+					election_id: id,
+					level: 'national',
+					area_code: 'NL',
+					area_name: 'Nederland',
+					party: p.Title,
+					votes: null,
+					seats: null
+				};
+				r.seats = v;
+				results.set(p.Title, r);
+			}
+		}
+		const list = [...results.values()].filter((r) => r.votes || r.seats);
+		const totalSeats = list.reduce((a, r) => a + (r.seats ?? 0), 0) || null;
+		rows += await save(
+			{
+				election_id: id,
+				country: 'NLD',
+				kind: 'TK',
+				date: KNOWN_DATES[year] ?? `${year}-01-01`,
+				name: `Tweede Kamer ${year}`,
+				source_id: 'cbs',
+				total_seats: totalSeats
+			},
+			list,
+			[turnout]
+		);
+	}
+	return rows;
+}
+
+/** US presidential election day: the Tuesday after the first Monday in November. */
+function usElectionDay(year: number) {
+	const d = new Date(Date.UTC(year, 10, 1));
+	const firstMonday = 1 + ((8 - d.getUTCDay()) % 7);
+	return `${year}-11-${String(firstMonday + 1).padStart(2, '0')}`;
+}
+
+async function mit(note: (w: string) => void) {
+	const ds = await getJson<{
+		data: { latestVersion: { files: { dataFile: { id: number; filename: string } }[] } };
+	}>(
+		'https://dataverse.harvard.edu/api/datasets/:persistentId/?persistentId=doi:10.7910/DVN/42MVDX'
+	);
+	const file = ds.data.latestVersion.files
+		.map((f) => f.dataFile)
+		.filter((f) => /^\d{4}-\d{4}-president\.(csv|tab)$/.test(f.filename))
+		.sort((a, b) => b.filename.localeCompare(a.filename))[0];
+	if (!file) throw new Error('MIT president file not found in doi:10.7910/DVN/42MVDX');
+	const text = await (
+		await http(`https://dataverse.harvard.edu/api/access/datafile/${file.id}`)
+	).text();
+	const sep = file.filename.endsWith('.tab') ? '\t' : ',';
+	const [header, ...lines] = parseCsv(text, sep);
+	const col = (name: string) => header.indexOf(name);
+	const [cYear, cState, cPo, cParty, cVotes, cTotal] = [
+		'year',
+		'state',
+		'state_po',
+		'party_simplified',
+		'candidatevotes',
+		'totalvotes'
+	].map(col);
+	if ([cYear, cState, cPo, cParty, cVotes, cTotal].some((i) => i < 0))
+		throw new Error(`MIT columns changed: ${header.join(',')}`);
+	const byYear = new Map<number, { results: Map<string, Result>; turnout: Map<string, Turnout> }>();
+	for (const l of lines) {
+		if (l.length < header.length - 1) continue;
+		const year = Number(l[cYear]);
+		const id = `US-PRES-${year}`;
+		const y = byYear.get(year) ?? { results: new Map(), turnout: new Map() };
+		byYear.set(year, y);
+		const state = l[cPo];
+		const party = l[cParty] || 'OTHER';
+		const votes = Number(l[cVotes]) || 0;
+		for (const [level, area_code, area_name] of [
+			['state', state, l[cState]],
+			['national', 'US', 'United States']
+		]) {
+			const key = `${level}|${area_code}|${party}`;
+			const r = y.results.get(key) ?? {
+				election_id: id,
+				level,
+				area_code,
+				area_name,
+				party,
+				votes: 0,
+				seats: null
+			};
+			r.votes = (r.votes ?? 0) + votes;
+			y.results.set(key, r);
+		}
+		if (!y.turnout.has(state))
+			y.turnout.set(state, {
+				election_id: id,
+				level: 'state',
+				area_code: state,
+				area_name: l[cState],
+				electorate: null,
+				ballots: Number(l[cTotal]) || null,
+				valid: null,
+				blank: null,
+				invalid: null
+			});
+	}
+	let rows = 0;
+	for (const [year, y] of byYear) {
+		const states = [...y.turnout.values()];
+		states.push({
+			election_id: `US-PRES-${year}`,
+			level: 'national',
+			area_code: 'US',
+			area_name: 'United States',
+			electorate: null,
+			ballots: states.reduce((a, s) => a + (s.ballots ?? 0), 0),
+			valid: null,
+			blank: null,
+			invalid: null
+		});
+		rows += await save(
+			{
+				election_id: `US-PRES-${year}`,
+				country: 'USA',
+				kind: 'PRES',
+				date: usElectionDay(year),
+				name: `US presidential ${year}`,
+				source_id: 'mitelection',
+				total_seats: null
+			},
+			[...y.results.values()],
+			states
+		);
+	}
+	if (!byYear.size) note('MIT file had no rows');
+	return rows;
+}
+
+export async function elections(opts: { force: boolean; only?: string; job: string }) {
+	const want = (k: string) => !opts.only || opts.only.split(',').includes(k);
+	if (want('kiesraad')) await run('kiesraad', opts.job, (note) => kiesraad(opts.force, note));
+	if (want('cbs')) await run('cbs', `${opts.job}:37278`, () => cbs37278());
+	if (want('mit')) await run('mitelection', opts.job, (note) => mit(note));
+}

scripts/stats/eurobarometer.ts

@@ -0,0 +1,135 @@
+// A16 Standard Eurobarometer, volume A (EU + candidate countries, weighted shares), via the
+// data.europa.eu catalogue (© European Union, reuse under the Commission's reuse policy).
+// Reads QA3 (most important issues facing our country), QA4 (… facing you personally), QA5
+// (… facing the EU) and the trust batteries (QA6_x national/international institutions, QA8_x
+// EU institutions; share "tend to trust"). Only waves published as xlsx; waves already loaded are
+// skipped unless --force.
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { getJson, http, insertChunks, log, run, sql } from './common';
+import { readXlsx, type Cell } from './xlsx';
+
+const SEARCH =
+	'https://data.europa.eu/api/hub/search/search?q=standard%20eurobarometer&limit=100&filter=dataset';
+
+type Dist = {
+	title?: string | Record<string, string>;
+	download_url?: string[];
+	access_url?: string[];
+};
+type Dataset = { title?: string | Record<string, string>; distributions?: Dist[] };
+const en = (t: Dataset['title']) =>
+	typeof t === 'string' ? t : (t?.en ?? Object.values(t ?? {})[0] ?? '');
+
+type Row = {
+	wave: string;
+	fieldwork_end: string | null;
+	topic: string;
+	qcode: string;
+	item: string;
+	country: string;
+	share: number;
+};
+
+const TOPICS: [RegExp, string][] = [
+	[/^QA3$/, 'mii_country'],
+	[/^QA4$/, 'mii_personal'],
+	[/^QA5$/, 'mii_eu'],
+	[/^QA6_\d+$/, 'trust'],
+	[/^QA8_\d+$/, 'trust_eu']
+];
+
+const str = (c: Cell | undefined) => (c == null ? '' : String(c)).trim();
+const share = (c: Cell | undefined) => {
+	const n = typeof c === 'number' ? c : Number(str(c));
+	return str(c) !== '' && Number.isFinite(n) ? n : null;
+};
+
+export function parseVolumeA(sheets: Awaited<ReturnType<typeof readXlsx>>): Row[] {
+	const content = sheets.find((s) => s.name === 'Content');
+	if (!content) throw new Error('no Content sheet');
+	const wave = str(content.rows.find((r) => /^wave/i.test(str(r[0])))?.[1]);
+	const fw = str(content.rows.find((r) => /^fieldwork/i.test(str(r[0])))?.[1]);
+	const d = fw.match(/(\d{1,2})\/(\d{1,2})\/(\d{4})\s*$/);
+	const fieldwork_end = d ? `${d[3]}-${d[2].padStart(2, '0')}-${d[1].padStart(2, '0')}` : null;
+	if (!wave) throw new Error('no wave in Content sheet');
+	const questions = new Map<string, string>();
+	for (const r of content.rows) {
+		const sheet = str(r[0])
+			.match(/^"?([A-Z][A-Z0-9_]*)"?\)?$|"([A-Z][A-Z0-9_]*)"\)\s*$/)
+			?.slice(1)
+			.find(Boolean);
+		if (sheet) questions.set(sheet, str(r[2]));
+	}
+	const out: Row[] = [];
+	for (const s of sheets) {
+		const topic = TOPICS.find(([re]) => re.test(s.name))?.[1];
+		if (!topic) continue;
+		const h = s.rows.findIndex(
+			(r) => r.some((c) => str(c) === 'BE') && r.some((c) => str(c) === 'NL')
+		);
+		if (h < 0) continue;
+		const countries = s.rows[h].map((c) => str(c).split('\n').at(-1) ?? '');
+		// Answers come in pairs: a French row with counts, then an English row with shares.
+		for (let i = h + 2; i < s.rows.length; i++) {
+			const label = str(s.rows[i][1]);
+			const next = s.rows[i + 1];
+			if (!label || !next || !str(next[1])) continue;
+			const values = countries.map((_, c) => share(next[c]));
+			if (!values.some((v) => v != null && v <= 1)) continue;
+			const item = str(next[1]);
+			i++;
+			let name = item;
+			if (topic.startsWith('trust')) {
+				if (!/^tend to trust/i.test(item)) continue;
+				name = (questions.get(s.name) ?? s.name).split(':-').at(-1)?.trim() ?? s.name;
+			}
+			countries.forEach((country, c) => {
+				const v = values[c];
+				if (c >= 2 && country && v != null && v <= 1)
+					out.push({ wave, fieldwork_end, topic, qcode: s.name, item: name, country, share: v });
+			});
+		}
+	}
+	return out;
+}
+
+export async function eurobarometer(opts: { force: boolean; job: string }) {
+	await run('eurobarometer', opts.job, async (note) => {
+		const res = await getJson<{ result: { results: Dataset[] } }>(SEARCH);
+		const waves = res.result.results
+			.filter((d) => /^standard eurobarometer/i.test(en(d.title)))
+			.flatMap((d) =>
+				(d.distributions ?? [])
+					.filter((x) => /volume[ _]A\.xlsx$/i.test(en(x.title)))
+					.map((x) => ({ dataset: en(d.title), url: x.download_url?.[0] ?? x.access_url?.[0] }))
+			)
+			.filter((w): w is { dataset: string; url: string } => !!w.url);
+		const loaded = new Set(
+			(await sql`select distinct wave from stats.eb_item`).map((r) => r.wave as string)
+		);
+		let rows = 0;
+		for (const w of waves) {
+			const num = w.dataset.match(/(?:STD|Eurobarometer )(\d{2,3})/i)?.[1];
+			if (num && !opts.force && [...loaded].some((l) => l.split('.')[0] === num)) continue;
+			const file = join(tmpdir(), `eb-${process.pid}.xlsx`);
+			await Bun.write(file, await (await http(w.url)).arrayBuffer());
+			let items: Row[] = [];
+			try {
+				items = parseVolumeA(await readXlsx(file));
+			} catch (error) {
+				note(`${w.dataset}: ${error instanceof Error ? error.message : error}`);
+			}
+			await Bun.file(file).delete();
+			if (!items.length) continue;
+			const dedup = new Map(items.map((r) => [`${r.wave}|${r.topic}|${r.item}|${r.country}`, r]));
+			await sql.begin(async (tx) => {
+				await tx`delete from stats.eb_item where wave = ${items[0].wave}`;
+				await insertChunks(tx, 'eb_item', [...dedup.values()]);
+			});
+			log(`eurobarometer ${items[0].wave}: ${dedup.size} rows`);
+			rows += dedup.size;
+		}
+		return rows;
+	});
+}

scripts/stats/gdelt.ts

@@ -0,0 +1,86 @@
+// A16 GDELT DOC 2.0 "timelinevol": daily share of monitored news coverage (by source country)
+// that mentions a GKG theme, last 30 days, upserted nightly. GDELT asks for at most one request
+// every 5 seconds, so the loop is deliberately slow (~12 minutes). GDELT data is open, cite GDELT.
+import { http, insertChunks, run, sql } from './common';
+
+export const GDELT_COUNTRIES: Record<string, string> = {
+	netherlands: 'NLD',
+	unitedstates: 'USA',
+	southkorea: 'KOR',
+	germany: 'DEU',
+	france: 'FRA',
+	unitedkingdom: 'GBR',
+	belgium: 'BEL',
+	japan: 'JPN',
+	india: 'IND',
+	brazil: 'BRA'
+};
+export const GDELT_THEMES = [
+	'ECON_INFLATION',
+	'ECON_HOUSING_PRICES',
+	'UNEMPLOYMENT',
+	'IMMIGRATION',
+	'REFUGEES',
+	'ENV_CLIMATECHANGE',
+	'TERROR',
+	'PROTEST',
+	'CORRUPTION',
+	'ELECTION',
+	'MILITARY',
+	'HEALTH_PANDEMIC'
+];
+
+export async function gdelt(opts: { job: string; timespan?: string }) {
+	await run('gdelt', opts.job, async (note) => {
+		let rows = 0;
+		const bad = new Set<string>();
+		for (const [name, iso3] of Object.entries(GDELT_COUNTRIES))
+			for (const theme of GDELT_THEMES) {
+				if (bad.has(theme)) continue;
+				const q = encodeURIComponent(`theme:${theme} sourcecountry:${name}`);
+				const url = `https://api.gdeltproject.org/api/v2/doc/doc?query=${q}&mode=timelinevol&format=json&timespan=${opts.timespan ?? '30d'}`;
+				let text: string;
+				try {
+					text = await (await http(url, {}, 3)).text();
+				} catch (error) {
+					note(`${theme}/${name}: ${error instanceof Error ? error.message : error}`);
+					continue;
+				}
+				let data: { timeline?: { data: { date: string; value: number }[] }[] };
+				try {
+					data = JSON.parse(text);
+				} catch {
+					// GDELT answers invalid queries with a plain-text message.
+					note(`${theme}: ${text.slice(0, 120)}`);
+					if (/theme|invalid/i.test(text)) bad.add(theme);
+					continue;
+				}
+				const points = (data.timeline?.[0]?.data ?? []).map((p) => ({
+					date: `${p.date.slice(0, 4)}-${p.date.slice(4, 6)}-${p.date.slice(6, 8)}`,
+					country: iso3,
+					theme,
+					vol_pct: p.value
+				}));
+				// Hourly resolution (short timespans): average per day.
+				const byDay = new Map<string, { sum: number; n: number; row: (typeof points)[number] }>();
+				for (const p of points) {
+					const d = byDay.get(p.date) ?? { sum: 0, n: 0, row: p };
+					d.sum += p.vol_pct;
+					d.n++;
+					byDay.set(p.date, d);
+				}
+				const daily = [...byDay.values()].map((d) => ({ ...d.row, vol_pct: d.sum / d.n }));
+				if (!daily.length) continue;
+				await sql.begin(async (tx) => {
+					await insertChunks(
+						tx,
+						'gdelt_theme_vol',
+						daily,
+						'on conflict (date, country, theme) do update set vol_pct = excluded.vol_pct'
+					);
+				});
+				rows += daily.length;
+			}
+		return rows;
+	});
+}

scripts/stats/gtrends.ts

@@ -0,0 +1,56 @@
+// A16 Google Trends "trending now" RSS (daily; terms of service: display with attribution).
+// Keeps each day's list per country; a re-run on the same day refreshes rank and traffic.
+import { getText, insertChunks, run, sql } from './common';
+import { decode } from './htmltable';
+
+const GEOS = ['NL', 'US', 'KR'];
+
+/** "200+", "2K+", "1M+" → lower bound as a number. */
+const traffic = (s: string | null) => {
+	const m = s?.replace(/,/g, '').match(/([\d.]+)\s*([KM])?/i);
+	if (!m) return null;
+	return Math.round(Number(m[1]) * (m[2]?.toUpperCase() === 'M' ? 1e6 : m[2] ? 1e3 : 1));
+};
+
+const tag = (xml: string, name: string) => {
+	const m = xml.match(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`));
+	return m ? decode(m[1].replace(/^<!\[CDATA\[|\]\]>$/g, '').trim()) : null;
+};
+
+export async function gtrends(opts: { job: string }) {
+	await run('gtrends', opts.job, async () => {
+		let rows = 0;
+		for (const geo of GEOS) {
+			const xml = await getText(`https://trends.google.com/trending/rss?geo=${geo}`);
+			const today = new Intl.DateTimeFormat('en-CA', { timeZone: 'Europe/Amsterdam' }).format(
+				new Date()
+			);
+			const items = [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m, i) => {
+				const news = [...m[1].matchAll(/<ht:news_item>([\s\S]*?)<\/ht:news_item>/g)]
+					.map((n) => ({
+						title: tag(n[1], 'ht:news_item_title'),
+						source: tag(n[1], 'ht:news_item_source'),
+						url: tag(n[1], 'ht:news_item_url')
+					}))
+					.slice(0, 3);
+				const pub = tag(m[1], 'pubDate');
+				return {
+					geo,
+					date: today,
+					rank: i + 1,
+					query: tag(m[1], 'title') ?? '',
+					approx_traffic: traffic(tag(m[1], 'ht:approx_traffic')),
+					pub_date: pub ? new Date(pub) : null,
+					news: JSON.stringify(news)
+				};
+			});
+			const unique = [...new Map(items.filter((x) => x.query).map((x) => [x.query, x])).values()];
+			await sql.begin(async (tx) => {
+				await tx`delete from stats.gt_trending where geo = ${geo} and date = ${today}`;
+				await insertChunks(tx, 'gt_trending', unique);
+			});
+			rows += unique.length;
+		}
+		return rows;
+	});
+}

scripts/stats/htmltable.ts

@@ -0,0 +1,129 @@
+// Minimal HTML table reader for MediaWiki output: expands rowspan/colspan into a grid of cell
+// texts (references, hidden spans and styles removed). Keeps each cell's data-sort-value.
+
+export type Cell = { text: string; sort: string | null; header: boolean };
+
+const ENTITIES: Record<string, string> = {
+	amp: '&',
+	lt: '<',
+	gt: '>',
+	quot: '"',
+	apos: "'",
+	nbsp: ' ',
+	ndash: '–',
+	mdash: '—',
+	minus: '−'
+};
+export const decode = (s: string) =>
+	s.replace(/&(#x?[0-9a-f]+|\w+);/gi, (m, e: string) => {
+		if (e[0] === '#')
+			return String.fromCodePoint(
+				e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1))
+			);
+		return ENTITIES[e.toLowerCase()] ?? m;
+	});
+
+export function cellText(html: string) {
+	return decode(
+		html
+			.replace(/<sup[\s\S]*?<\/sup>/gi, '')
+			.replace(/<style[\s\S]*?<\/style>/gi, '')
+			.replace(/<span[^>]*display:\s*none[^>]*>[\s\S]*?<\/span>/gi, '')
+			.replace(/<br\s*\/?>/gi, ' ')
+			.replace(/<[^>]+>/g, '')
+	)
+		.replace(/\s+/g, ' ')
+		.trim();
+}
+
+/** All tables (outer HTML) whose opening tag matches `cls`, with the text of the nearest
+ * preceding heading. Nested tables are skipped over. */
+export function findTables(html: string, cls = /wikitable/) {
+	const out: { html: string; heading: string }[] = [];
+	const re = /<table\b[^>]*>/gi;
+	let m: RegExpExecArray | null;
+	while ((m = re.exec(html))) {
+		if (!cls.test(m[0])) continue;
+		let depth = 1;
+		const tag = /<\/?table\b[^>]*>/gi;
+		tag.lastIndex = m.index + m[0].length;
+		let t: RegExpExecArray | null;
+		while (depth && (t = tag.exec(html))) depth += t[0][1] === '/' ? -1 : 1;
+		const end = tag.lastIndex;
+		const headings = [...html.slice(0, m.index).matchAll(/<h[2-4][^>]*>([\s\S]*?)<\/h[2-4]>/gi)];
+		out.push({ html: html.slice(m.index, end), heading: cellText(headings.at(-1)?.[1] ?? '') });
+		re.lastIndex = end;
+	}
+	return out;
+}
+
+export function tableGrid(table: string): Cell[][] {
+	const grid: Cell[][] = [];
+	const pending: (Cell & { left: number })[] = []; // by column: rowspans carried down
+	const rows = table.split(/<tr\b[^>]*>/i).slice(1);
+	for (const rowHtml of rows) {
+		const row: Cell[] = [];
+		let col = 0;
+		const place = () => {
+			while (pending[col] && pending[col].left > 0) {
+				row[col] = pending[col];
+				pending[col].left--;
+				col++;
+			}
+		};
+		for (const c of rowHtml.matchAll(/<(t[hd])\b([^>]*)>([\s\S]*?)(?=<t[hd]\b|<\/tr>|$)/gi)) {
+			place();
+			const attrs = c[2];
+			const rowspan = Number(attrs.match(/rowspan="?(\d+)/i)?.[1] ?? 1);
+			const colspan = Number(attrs.match(/colspan="?(\d+)/i)?.[1] ?? 1);
+			const cell: Cell = {
+				text: cellText(c[3].replace(/<\/t[hd]>[\s\S]*$/i, '')),
+				sort: attrs.match(/data-sort-value="([^"]*)"/i)?.[1] ?? null,
+				header: c[1].toLowerCase() === 'th'
+			};
+			for (let k = 0; k < colspan; k++) {
+				row[col] = cell;
+				if (rowspan > 1) pending[col] = { ...cell, left: rowspan - 1 };
+				else delete pending[col];
+				col++;
+			}
+		}
+		place();
+		if (row.length) grid.push(row);
+	}
+	return grid;
+}
+
+const MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
+
+/** End date of a fieldwork range like "13–15 Oct 2026", "Sep 28 – Oct 2, 2026", "2026-10-15". */
+export function fieldworkEnd(
+	text: string,
+	sort: string | null,
+	fallbackYear?: number
+): string | null {
+	for (const s of [sort, text]) {
+		const iso = s?.match(/(\d{4})-(\d{1,2})-(\d{1,2})(?!.*\d{4}-\d{1,2}-\d{1,2})/);
+		if (iso) return `${iso[1]}-${iso[2].padStart(2, '0')}-${iso[3].padStart(2, '0')}`;
+	}
+	const t = text.toLowerCase().replace(/[–—−]/g, '-');
+	const year = Number(t.match(/(\d{4})(?!.*\d{4})/)?.[1] ?? fallbackYear);
+	if (!year) return null;
+	const months = [...t.matchAll(/\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?/g)];
+	if (!months.length) return null;
+	const last = months[months.length - 1];
+	const month = MONTHS.indexOf(last[1]) + 1;
+	// Day: the number right after the last month ("Oct 2") or right before it ("15 Oct").
+	const after = t
+		.slice(last.index! + last[0].length)
+		.match(/^\s*(\d{1,2})(?:\s*-\s*(\d{1,2}))?\b(?!\d)/);
+	const before = t.slice(0, last.index).match(/(\d{1,2})\s*$/);
+	const day = Number(after?.[2] ?? after?.[1] ?? before?.[1] ?? 0);
+	if (!day || day > 31) return null;
+	return `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
+}
+
+export const firstNumber = (s: string) => {
+	const m = s.replace(/,(?=\d{3}\b)/g, '').match(/-?\d+(?:\.\d+)?/);
+	return m ? Number(m[0]) : null;
+};

scripts/stats/owid.ts

@@ -56,7 +56,7 @@ export const ENERGY_COLUMNS: Record<string, { title: string; unit: string; serie
 };
 
 /** Minimal CSV parser (quoted fields, commas inside quotes). */
-export function parseCsv(text: string): string[][] {
+export function parseCsv(text: string, sep = ','): string[][] {
 	const out: string[][] = [];
 	let row: string[] = [];
 	let field = '';
@@ -70,7 +70,7 @@ export function parseCsv(text: string): string[][] {
 			} else if (ch === '"') quoted = false;
 			else field += ch;
 		} else if (ch === '"') quoted = true;
-		else if (ch === ',') {
+		else if (ch === sep) {
 			row.push(field);
 			field = '';
 		} else if (ch === '\n' || ch === '\r') {

scripts/stats/parties.ts

@@ -0,0 +1,248 @@
+// A15 party positions: CHES trend file (party means 1999–2024) and V-Party v2 (V-Dem, via the
+// vdemdata GitHub repo's RData), linked through the Party Facts crosswalk. Writes stats.wo_party
+// (data-driven rows; the curated guide parties keep their own `wo:` rows) and
+// stats.wo_party_position. Curated rows get a partyfacts_id when a match is unambiguous.
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { insertChunks, http, log, run, sql } from './common';
+import { parseCsv } from './owid';
+
+const CHES_URL =
+	'https://github.com/chesdata/chesdata.github.io/releases/download/ches-trend/1999-2024_CHES_dataset_meansV2.csv';
+const VPARTY_URL = 'https://github.com/vdeminstitute/vdemdata/raw/master/data/vparty.RData';
+const PARTYFACTS_URL = 'https://partyfacts.herokuapp.com/download/external-parties-csv/';
+const PYTHON = process.env.STATS_PYTHON ?? `${process.env.HOME}/.local/share/stats-py/bin/python`;
+
+const CHES_COUNTRY: Record<number, string> = {
+	1: 'BEL',
+	2: 'DNK',
+	3: 'DEU',
+	4: 'GRC',
+	5: 'ESP',
+	6: 'FRA',
+	7: 'IRL',
+	8: 'ITA',
+	10: 'NLD',
+	11: 'GBR',
+	12: 'PRT',
+	13: 'AUT',
+	14: 'FIN',
+	16: 'SWE',
+	20: 'BGR',
+	21: 'CZE',
+	22: 'EST',
+	23: 'HUN',
+	24: 'LVA',
+	25: 'LTU',
+	26: 'POL',
+	27: 'ROU',
+	28: 'SVK',
+	29: 'SVN',
+	31: 'HRV',
+	37: 'MLT',
+	38: 'LUX',
+	40: 'CYP',
+	9: 'NOR',
+	35: 'NOR',
+	36: 'CHE',
+	34: 'TUR',
+	45: 'ISL'
+};
+const CHES_FAMILY: Record<number, string> = {
+	1: 'radical right',
+	2: 'conservative',
+	3: 'liberal',
+	4: 'christian democratic',
+	5: 'socialist',
+	6: 'radical left',
+	7: 'green',
+	8: 'regionalist',
+	9: 'no family',
+	10: 'confessional',
+	11: 'agrarian/centre'
+};
+const CHES_DIMS = ['lrgen', 'lrecon', 'galtan', 'eu_position', 'immigrate_policy'];
+const VPARTY_DIMS = [
+	'v2xpa_popul',
+	'v2xpa_antiplural',
+	'v2pariglef',
+	'v2paimmig',
+	'v2palgbt',
+	'v2parelig',
+	'v2pawelf',
+	'v2paculsup'
+];
+
+type Party = {
+	party_id: string;
+	iso3: string | null;
+	name: string | null;
+	abbrev: string | null;
+	family: string | null;
+	ches_id: number | null;
+	manifesto_id: number | null;
+	vparty_id: number | null;
+	partyfacts_id: number | null;
+	sort: null;
+};
+type Position = {
+	party_id: string;
+	year: number;
+	source: string;
+	dim: string;
+	value: number | null;
+	vote_share: number | null;
+};
+
+const table = (text: string) => {
+	const [header, ...rows] = parseCsv(text);
+	return rows
+		.filter((r) => r.length >= header.length - 1)
+		.map((r) => Object.fromEntries(header.map((h, i) => [h, r[i] ?? ''])));
+};
+const num = (s: string | undefined) =>
+	s == null || s === '' || s === 'NA' ? null : Number.isFinite(Number(s)) ? Number(s) : null;
+
+export async function parties(opts: { job: string }) {
+	await run('ches', opts.job, async (note) => {
+		// Party Facts: dataset ids → partyfacts ids (and countries).
+		const pf = table(await (await http(PARTYFACTS_URL)).text());
+		const link = new Map<string, { pf: number | null; country: string }>();
+		for (const r of pf)
+			if (r.dataset_key === 'ches' || r.dataset_key === 'vparty' || r.dataset_key === 'manifesto')
+				link.set(`${r.dataset_key}:${r.dataset_party_id}`, {
+					pf: num(r.partyfacts_id),
+					country: r.country
+				});
+		const partiesById = new Map<string, Party>();
+		const upsert = (p: Partial<Party> & { party_id: string }) => {
+			const prev = partiesById.get(p.party_id);
+			if (!prev)
+				partiesById.set(p.party_id, {
+					iso3: null,
+					name: null,
+					abbrev: null,
+					family: null,
+					ches_id: null,
+					manifesto_id: null,
+					vparty_id: null,
+					partyfacts_id: null,
+					sort: null,
+					...p
+				});
+			else
+				for (const [k, v] of Object.entries(p))
+					if (v != null && (prev as Record<string, unknown>)[k] == null)
+						(prev as Record<string, unknown>)[k] = v;
+		};
+		const positions = new Map<string, Position>();
+		const put = (p: Position) => positions.set(`${p.party_id}|${p.year}|${p.source}|${p.dim}`, p);
+
+		// CHES
+		const ches = table(await (await http(CHES_URL)).text());
+		for (const r of ches) {
+			const chesId = num(r.party_id);
+			if (chesId == null) continue;
+			const l = link.get(`ches:${chesId}`);
+			const id = l?.pf != null ? `pf:${l.pf}` : `ches:${chesId}`;
+			const iso3 = CHES_COUNTRY[Number(r.country)] ?? l?.country ?? null;
+			if (!iso3) note(`CHES country ${r.country} unmapped`);
+			upsert({
+				party_id: id,
+				iso3,
+				abbrev: r.party || null,
+				family: CHES_FAMILY[Number(r.family)] ?? null,
+				ches_id: chesId,
+				manifesto_id: num(r.cmp_id),
+				partyfacts_id: l?.pf ?? null
+			});
+			for (const dim of CHES_DIMS)
+				if (r[dim] !== undefined)
+					put({
+						party_id: id,
+						year: Number(r.year),
+						source: 'ches',
+						dim,
+						value: num(r[dim]),
+						vote_share: num(r.vote)
+					});
+		}
+		const chesRows = positions.size;
+
+		// V-Party (RData → CSV via the stats Python venv)
+		let vpartyRows = 0;
+		try {
+			const file = join(tmpdir(), `vparty-${process.pid}.RData`);
+			await Bun.write(file, await (await http(VPARTY_URL)).arrayBuffer());
+			const proc = Bun.spawn([PYTHON, join(import.meta.dir, 'vparty-csv.py'), file], {
+				stdout: 'pipe',
+				stderr: 'pipe'
+			});
+			const csv = await new Response(proc.stdout).text();
+			const err = await new Response(proc.stderr).text();
+			if ((await proc.exited) !== 0) throw new Error(`vparty-csv.py failed: ${err.slice(0, 300)}`);
+			await Bun.file(file).delete();
+			for (const r of table(csv)) {
+				const vpId = num(r.v2paid);
+				if (vpId == null) continue;
+				const pfId = num(r.pf_party_id) ?? link.get(`vparty:${vpId}`)?.pf ?? null;
+				const id = pfId != null ? `pf:${pfId}` : `vp:${vpId}`;
+				upsert({
+					party_id: id,
+					iso3: r.country_text_id || null,
+					name: r.v2paenname || null,
+					abbrev: r.v2pashname || null,
+					vparty_id: vpId,
+					partyfacts_id: pfId,
+					ches_id: num(r.CHES_ID)
+				});
+				for (const dim of VPARTY_DIMS)
+					if (r[dim] !== undefined && r[dim] !== 'NA') {
+						put({
+							party_id: id,
+							year: Number(r.year),
+							source: 'vparty',
+							dim,
+							value: num(r[dim]),
+							vote_share: num(r.v2pavote)
+						});
+						vpartyRows++;
+					}
+			}
+		} catch (error) {
+			note(`V-Party skipped: ${error instanceof Error ? error.message : error}`);
+		}
+
+		// Names for CHES-only parties from Party Facts.
+		for (const r of pf)
+			if (r.dataset_key === 'ches') {
+				const id =
+					num(r.partyfacts_id) != null ? `pf:${r.partyfacts_id}` : `ches:${r.dataset_party_id}`;
+				const p = partiesById.get(id);
+				if (p && !p.name) p.name = r.name_english || r.name || null;
+			}
+
+		const list = [...partiesById.values()];
+		await sql.begin(async (tx) => {
+			await tx`delete from stats.wo_party_position where source in ('ches', 'vparty')`;
+			await tx`delete from stats.wo_party where party_id not like 'wo:%'`;
+			await insertChunks(tx, 'wo_party', list);
+			await insertChunks(tx, 'wo_party_position', [...positions.values()]);
+			// Link curated guide parties to Party Facts where exactly one data party in the same
+			// country has the same abbreviation (case-insensitive).
+			await tx`
+				update stats.wo_party w set partyfacts_id = m.partyfacts_id
+				from (
+					select c.party_id, min(d.partyfacts_id) as partyfacts_id
+					from stats.wo_party c
+					join stats.wo_party d on d.iso3 = c.iso3 and d.party_id not like 'wo:%'
+						and lower(d.abbrev) = lower(c.abbrev) and d.partyfacts_id is not null
+					where c.party_id like 'wo:%'
+					group by c.party_id having count(distinct d.partyfacts_id) = 1
+				) m
+				where w.party_id = m.party_id`;
+		});
+		log(`parties: ${list.length} parties, ${chesRows} CHES + ${vpartyRows} V-Party positions`);
+		return list.length + positions.size;
+	});
+}

scripts/stats/remotezip.ts

@@ -0,0 +1,123 @@
+// Read single entries from a remote zip with HTTP range requests, so a 25 MB archive costs only
+// its central directory plus the files we need. Falls back to one full download when the server
+// ignores ranges. Handles stored and deflated entries (no zip64: the archives we read are small).
+import { inflateRawSync } from 'node:zlib';
+import { http } from './common';
+
+export type ZipEntry = {
+	name: string;
+	method: number;
+	compSize: number;
+	size: number;
+	offset: number;
+};
+
+export class RemoteZip {
+	private full?: Uint8Array;
+	private constructor(
+		readonly url: string,
+		readonly size: number,
+		readonly entries: ZipEntry[]
+	) {}
+
+	private static async range(url: string, start: number, end: number, size: number) {
+		const res = await http(url, {
+			headers: { range: `bytes=${start}-${end}` },
+			signal: AbortSignal.timeout(1_200_000)
+		});
+		const buf = new Uint8Array(await res.arrayBuffer());
+		if (res.status === 206) return { buf, whole: null as Uint8Array | null };
+		if (buf.length !== size) throw new Error(`short download of ${url}: ${buf.length}/${size}`);
+		return { buf: buf.subarray(start, end + 1), whole: buf };
+	}
+
+	static async open(url: string): Promise<RemoteZip> {
+		const head = await http(url, { method: 'HEAD' });
+		const size = Number(head.headers.get('content-length'));
+		if (!size) throw new Error(`no content-length for ${url}`);
+		const tailLen = Math.min(size, 66000);
+		const tail = await RemoteZip.range(url, size - tailLen, size - 1, size);
+		const t = tail.buf;
+		const v = new DataView(t.buffer, t.byteOffset, t.byteLength);
+		let eocd = -1;
+		for (let i = t.length - 22; i >= 0; i--)
+			if (v.getUint32(i, true) === 0x06054b50) {
+				eocd = i;
+				break;
+			}
+		if (eocd < 0) throw new Error(`not a zip: ${url}`);
+		const cdSize = v.getUint32(eocd + 12, true);
+		const cdOffset = v.getUint32(eocd + 16, true);
+		const cd = tail.whole
+			? tail.whole.subarray(cdOffset, cdOffset + cdSize)
+			: (await RemoteZip.range(url, cdOffset, cdOffset + cdSize - 1, size)).buf;
+		const c = new DataView(cd.buffer, cd.byteOffset, cd.byteLength);
+		const entries: ZipEntry[] = [];
+		const dec = new TextDecoder();
+		for (let p = 0; p + 46 <= cd.length && c.getUint32(p, true) === 0x02014b50;) {
+			const nameLen = c.getUint16(p + 28, true);
+			const extraLen = c.getUint16(p + 30, true);
+			const commentLen = c.getUint16(p + 32, true);
+			entries.push({
+				method: c.getUint16(p + 10, true),
+				compSize: c.getUint32(p + 20, true),
+				size: c.getUint32(p + 24, true),
+				offset: c.getUint32(p + 42, true),
+				name: dec.decode(cd.subarray(p + 46, p + 46 + nameLen))
+			});
+			p += 46 + nameLen + extraLen + commentLen;
+		}
+		const zip = new RemoteZip(url, size, entries);
+		if (tail.whole) zip.full = tail.whole;
+		return zip;
+	}
+
+	find(pattern: RegExp) {
+		return this.entries.filter((e) => pattern.test(e.name) && !e.name.endsWith('/'));
+	}
+
+	async read(entry: ZipEntry): Promise<Uint8Array> {
+		let local: Uint8Array;
+		if (this.full)
+			local = this.full.subarray(entry.offset, entry.offset + 30 + 65535 * 2 + entry.compSize);
+		else
+			local = (
+				await RemoteZip.range(
+					this.url,
+					entry.offset,
+					Math.min(
+						this.size - 1,
+						entry.offset + 30 + 1024 + entry.name.length * 4 + entry.compSize
+					),
+					this.size
+				)
+			).buf;
+		const v = new DataView(local.buffer, local.byteOffset, local.byteLength);
+		if (v.getUint32(0, true) !== 0x04034b50) throw new Error(`bad local header for ${entry.name}`);
+		const start = 30 + v.getUint16(26, true) + v.getUint16(28, true);
+		if (start + entry.compSize > local.length) {
+			// Unusually long local extra field: fetch exactly.
+			const exact = await RemoteZip.range(
+				this.url,
+				entry.offset + start,
+				entry.offset + start + entry.compSize - 1,
+				this.size
+			);
+			return this.inflate(entry, exact.buf);
+		}
+		return this.inflate(entry, local.subarray(start, start + entry.compSize));
+	}
+
+	private inflate(entry: ZipEntry, data: Uint8Array) {
+		if (entry.method === 0) return data;
+		if (entry.method !== 8)
+			throw new Error(`unsupported zip method ${entry.method} (${entry.name})`);
+		const out = new Uint8Array(inflateRawSync(data));
+		if (out.length !== entry.size) throw new Error(`size mismatch for ${entry.name}`);
+		return out;
+	}
+
+	async text(entry: ZipEntry, encoding = 'utf-8') {
+		return new TextDecoder(encoding).decode(await this.read(entry)).replace(/^\uFEFF/, '');
+	}
+}

scripts/stats/vparty-csv.py

@@ -0,0 +1,14 @@
+# Convert V-Dem's vparty.RData to CSV on stdout (selected columns only). Run with the stats
+# Python venv (pyreadr + pandas): ~/.local/share/stats-py/bin/python vparty-csv.py file.RData
+import sys
+
+import pyreadr
+
+COLS = [
+    "v2paid", "pf_party_id", "CHES_ID", "country_text_id", "year", "v2paenname", "v2pashname",
+    "v2pavote", "v2paseatshare", "v2xpa_popul", "v2xpa_antiplural", "v2pariglef", "v2paimmig",
+    "v2palgbt", "v2parelig", "v2pawelf", "v2paculsup",
+]
+frames = pyreadr.read_r(sys.argv[1])
+df = next(iter(frames.values()))
+df[[c for c in COLS if c in df.columns]].to_csv(sys.stdout, index=False)

scripts/stats/whr.ts

@@ -0,0 +1,124 @@
+// A16 World Happiness Report: Figure 2.1 data (life evaluation, 3-year average, and the
+// explained components), all years 2011–latest. Tries the newest edition first.
+// © Wellbeing Research Centre / Gallup World Poll data, reuse with attribution.
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { http, insertChunks, log, run, sql } from './common';
+import { readXlsx, type Cell } from './xlsx';
+
+const NAME_ISO3: Record<string, string | null> = {
+	'Côte d’Ivoire': 'CIV',
+	"Côte d'Ivoire": 'CIV',
+	'Ivory Coast': 'CIV',
+	'DR Congo': 'COD',
+	'Hong Kong SAR of China': 'HKG',
+	'Hong Kong S.A.R. of China': 'HKG',
+	Kosovo: 'XKX',
+	'Lao PDR': 'LAO',
+	Laos: 'LAO',
+	'Republic of Korea': 'KOR',
+	'South Korea': 'KOR',
+	'Republic of Moldova': 'MDA',
+	'Russian Federation': 'RUS',
+	'State of Palestine': 'PSE',
+	Swaziland: 'SWZ',
+	Eswatini: 'SWZ',
+	'Taiwan Province of China': 'TWN',
+	Türkiye: 'TUR',
+	Turkiye: 'TUR',
+	'Viet Nam': 'VNM',
+	'North Cyprus': null,
+	'Somaliland Region': null,
+	Czechia: 'CZE',
+	'Congo (Brazzaville)': 'COG',
+	'Congo (Kinshasa)': 'COD'
+};
+
+const n = (c: Cell | undefined) =>
+	typeof c === 'number'
+		? c
+		: c == null || c === ''
+			? null
+			: Number.isFinite(Number(c))
+				? Number(c)
+				: null;
+
+export async function whr(opts: { job: string }) {
+	await run('whr', opts.job, async (note) => {
+		const yy = new Date().getFullYear() % 100;
+		let file: string | null = null;
+		for (const e of [yy + 1, yy, yy - 1]) {
+			try {
+				const res = await http(
+					`https://files.worldhappiness.report/WHR${e}_Data_Figure_2.1.xlsx`,
+					{},
+					1
+				);
+				file = join(tmpdir(), `whr-${process.pid}.xlsx`);
+				await Bun.write(file, await res.arrayBuffer());
+				log(`whr: edition 20${e}`);
+				break;
+			} catch {
+				// try an older edition
+			}
+		}
+		if (!file) throw new Error('no WHR Figure 2.1 file found');
+		const [sheet] = await readXlsx(file);
+		await Bun.file(file).delete();
+		const [header, ...rows] = sheet.rows;
+		const col = (re: RegExp) => header.findIndex((h) => re.test(String(h ?? '')));
+		const c = {
+			year: col(/^Year/),
+			rank: col(/^Rank/),
+			country: col(/^Country/),
+			ladder: col(/^Life evaluation|^Ladder/),
+			lower: col(/^Lower/),
+			upper: col(/^Upper/),
+			gdp: col(/GDP/),
+			social: col(/Social/),
+			health: col(/Healthy/),
+			freedom: col(/Freedom/),
+			generosity: col(/Generosity/),
+			corruption: col(/corruption/i),
+			dystopia: col(/Dystopia/)
+		};
+		if (c.year < 0 || c.country < 0 || c.ladder < 0)
+			throw new Error(`WHR columns changed: ${header.join('|')}`);
+		const iso = new Map(
+			(await sql`select distinct country, iso3 from stats.owid_energy where length(iso3) = 3`).map(
+				(r) => [r.country as string, r.iso3 as string]
+			)
+		);
+		const unmapped = new Set<string>();
+		const out = rows
+			.filter((r) => n(r[c.year]) && r[c.country])
+			.map((r) => {
+				const country = String(r[c.country]);
+				const iso3 = country in NAME_ISO3 ? NAME_ISO3[country] : (iso.get(country) ?? null);
+				if (!iso3 && !(country in NAME_ISO3)) unmapped.add(country);
+				const v = (k: keyof typeof c) => (c[k] >= 0 ? n(r[c[k]]) : null);
+				return {
+					year: v('year')!,
+					country,
+					iso3,
+					rank: v('rank'),
+					ladder: v('ladder'),
+					lower: v('lower'),
+					upper: v('upper'),
+					gdp: v('gdp'),
+					social: v('social'),
+					health: v('health'),
+					freedom: v('freedom'),
+					generosity: v('generosity'),
+					corruption: v('corruption'),
+					dystopia: v('dystopia')
+				};
+			});
+		if (unmapped.size) note(`no ISO3 for: ${[...unmapped].join(', ')}`);
+		await sql.begin(async (tx) => {
+			await tx`delete from stats.whr_score`;
+			await insertChunks(tx, 'whr_score', out);
+		});
+		return out.length;
+	});
+}

scripts/stats/wikipolls.ts

@@ -0,0 +1,195 @@
+// A16 Wikipedia poll tables (CC BY-SA 4.0). For each series we fetch the rendered page via the
+// MediaWiki API, keep the wikitext snapshot per revision (stats.op_wiki_snapshot), and upsert
+// one row per poll and option into stats.op_poll. Polls dropped from the page stay in the DB.
+import { getJson, insertChunks, log, run, sql } from './common';
+import { fieldworkEnd, findTables, firstNumber, tableGrid, type Cell } from './htmltable';
+
+const API = 'https://en.wikipedia.org/w/api.php';
+
+type Series = {
+	series: string;
+	page: string;
+	/** Header labels that identify the table(s) to read. */
+	match: string[];
+	/** How many matching tables to read (newest first on these pages). */
+	tables: number;
+	pollster: RegExp;
+	date: RegExp;
+	sample?: RegExp;
+	skip: RegExp;
+};
+
+export const WIKI_SERIES: Series[] = [
+	{
+		series: 'nl-tk-seats',
+		page: 'Opinion_polling_for_the_next_Dutch_general_election',
+		match: ['Polling firm', 'Fieldwork date'],
+		tables: 2,
+		pollster: /^Polling firm/i,
+		date: /^Fieldwork/i,
+		sample: /^Sample/i,
+		skip: /^(Lead|Ref|Ref\.)$/i
+	},
+	{
+		series: 'us-trump-approval-aggregate',
+		page: 'Opinion_polling_on_the_second_Trump_presidency',
+		match: ['Aggregator', 'Approve'],
+		tables: 1,
+		pollster: /^Aggregator/i,
+		date: /^Updated/i,
+		skip: /^(Lead|Net)$/i
+	},
+	{
+		series: 'us-trump-approval',
+		page: 'Opinion_polling_on_the_second_Trump_presidency',
+		match: ['Poll source', 'Approve'],
+		tables: 13,
+		pollster: /^Poll source/i,
+		date: /^Date/i,
+		sample: /^Sample/i,
+		skip: /^(MoE|Net|Lead)$/i
+	},
+	{
+		series: 'kr-lee-approval',
+		page: 'Opinion_polling_on_the_Lee_Jae_Myung_presidency',
+		match: ['Polling firm', 'Approve'],
+		tables: 2,
+		pollster: /^Polling firm/i,
+		date: /^Fieldwork/i,
+		sample: /^Sample/i,
+		skip: /^(Margin|Net|Lead)/i
+	},
+	{
+		series: 'kr-party',
+		page: 'Opinion_polling_for_the_2028_South_Korean_legislative_election',
+		match: ['Polling firm', 'DPK'],
+		tables: 2,
+		pollster: /^Polling firm/i,
+		date: /^Fieldwork/i,
+		sample: /^Sample/i,
+		skip: /^(Margin|Lead)/i
+	}
+];
+
+type Poll = {
+	series: string;
+	poll_key: string;
+	fieldwork_end: string;
+	pollster: string;
+	sample: number | null;
+	option: string;
+	value: number;
+	seen_at: Date;
+};
+
+export function pollsFromHtml(
+	s: Series,
+	html: string,
+	today = new Date()
+): Omit<Poll, 'seen_at'>[] {
+	const out: Omit<Poll, 'seen_at'>[] = [];
+	const tables = findTables(html)
+		.map((t) => ({
+			g: tableGrid(t.html),
+			year: Number(t.heading.match(/\b(19|20)\d{2}\b/)?.[0] ?? today.getFullYear())
+		}))
+		.filter(({ g }) => {
+			const head = (g[0] ?? []).map((c) => c.text);
+			return s.match.every((m) => head.some((h) => h.startsWith(m)));
+		})
+		.slice(0, s.tables);
+	const limit = new Date(today.getTime() + 7 * 864e5).toISOString().slice(0, 10);
+	for (const { g, year } of tables) {
+		const head = g[0].map((c) => c.text.replace(/\s*\/\s*/g, '/'));
+		const col = (re: RegExp) => head.findIndex((h) => re.test(h));
+		const [cPoll, cDate, cSample] = [col(s.pollster), col(s.date), s.sample ? col(s.sample) : -1];
+		const options = head
+			.map((h, i) => ({ h, i }))
+			.filter(
+				({ h, i }) =>
+					i !== cPoll &&
+					i !== cDate &&
+					i !== cSample &&
+					h &&
+					!s.skip.test(h) &&
+					head.indexOf(h) === i
+			);
+		for (const row of g.slice(1)) {
+			if (row.every((c) => c.header) || row.length < head.length - 2) continue;
+			const dateCell: Cell | undefined = row[cDate];
+			if (!dateCell) continue;
+			let end = fieldworkEnd(dateCell.text, dateCell.sort, year);
+			if (end && end > limit) end = `${Number(end.slice(0, 4)) - 1}${end.slice(4)}`;
+			if (!end) continue;
+			const pollster = row[cPoll]?.text ?? '';
+			if (!pollster || pollster === dateCell.text) continue;
+			const sample = cSample >= 0 ? firstNumber(row[cSample]?.text ?? '') : null;
+			const key = `${end}|${pollster}|${sample ?? ''}`;
+			for (const { h, i } of options) {
+				const v = firstNumber(row[i]?.text ?? '');
+				if (v != null && !row[i]?.header)
+					out.push({
+						series: s.series,
+						poll_key: key,
+						fieldwork_end: end,
+						pollster,
+						sample,
+						option: h,
+						value: v
+					});
+			}
+		}
+	}
+	// Same poll twice (repeated across tables): keep the first.
+	const seen = new Set<string>();
+	return out.filter((p) => {
+		const k = `${p.poll_key}|${p.option}`;
+		if (seen.has(k)) return false;
+		seen.add(k);
+		return true;
+	});
+}
+
+export async function wikipolls(opts: { job: string }) {
+	await run('wikipedia', opts.job, async (note) => {
+		const pages = new Map<string, { revid: number; html: string; wikitext: string }>();
+		let rows = 0;
+		for (const s of WIKI_SERIES) {
+			let page = pages.get(s.page);
+			if (!page) {
+				const url = `${API}?action=parse&page=${encodeURIComponent(s.page)}&prop=text|wikitext|revid&redirects=1&format=json&formatversion=2`;
+				const res = await getJson<{
+					parse?: { revid: number; text: string; wikitext: string };
+					error?: { info: string };
+				}>(url);
+				if (!res.parse) {
+					note(`${s.page}: ${res.error?.info ?? 'no parse result'}`);
+					continue;
+				}
+				page = { revid: res.parse.revid, html: res.parse.text, wikitext: res.parse.wikitext };
+				pages.set(s.page, page);
+				await sql`
+					insert into stats.op_wiki_snapshot (page, revid, fetched_at, wikitext)
+					values (${s.page}, ${page.revid}, now(), ${page.wikitext})
+					on conflict (page, revid) do nothing`;
+			}
+			const polls = pollsFromHtml(s, page.html);
+			if (!polls.length) {
+				note(`${s.series}: no polls parsed from ${s.page} rev ${page.revid}`);
+				continue;
+			}
+			const now = new Date();
+			await sql.begin(async (tx) => {
+				await insertChunks(
+					tx,
+					'op_poll',
+					polls.map((p) => ({ ...p, seen_at: now })),
+					'on conflict (series, poll_key, option) do update set value = excluded.value, seen_at = excluded.seen_at'
+				);
+			});
+			log(`wikipolls ${s.series}: ${polls.length} values`);
+			rows += polls.length;
+		}
+		return rows;
+	});
+}

src/lib/server/db/schema/stats.ts

@@ -467,3 +467,280 @@ export const statsCwCbcsBop = statsSchema.table(
 	},
 	(t) => [primaryKey({ columns: [t.period, t.item] })]
 );
+
+// ── Step 3: elections (A14) ──────────────────────────────────────────────────────────────
+
+/** One election: NL Tweede Kamer (Kiesraad, CBS 37278) or US presidential (MIT Election Lab). */
+export const statsElecElection = statsSchema.table('elec_election', {
+	/** NL-TK-2025, US-PRES-2024, … */
+	electionId: text('election_id').primaryKey(),
+	/** ISO 3166-1 alpha-3. */
+	country: char('country', { length: 3 }).notNull(),
+	/** TK (Tweede Kamer), PRES (US president). */
+	kind: text('kind').notNull(),
+	date: date('date', { mode: 'string' }).notNull(),
+	name: text('name').notNull(),
+	sourceId: text('source_id').notNull(),
+	totalSeats: integer('total_seats')
+});
+
+export const statsElecResult = statsSchema.table(
+	'elec_result',
+	{
+		electionId: text('election_id').notNull(),
+		/** national, province (kieskring not used), muni, station, state. */
+		level: text('level').notNull(),
+		/** NL / GM0513 / GM0513-SB101 / TX … */
+		areaCode: text('area_code').notNull(),
+		areaName: text('area_name'),
+		/** List name as the source spells it (normalised to a short label on the page). */
+		party: text('party').notNull(),
+		votes: bigint('votes', { mode: 'number' }),
+		seats: integer('seats')
+	},
+	(t) => [
+		primaryKey({ columns: [t.electionId, t.level, t.areaCode, t.party] }),
+		index('elec_result_area_idx').on(t.level, t.areaCode)
+	]
+);
+
+export const statsElecTurnout = statsSchema.table(
+	'elec_turnout',
+	{
+		electionId: text('election_id').notNull(),
+		level: text('level').notNull(),
+		areaCode: text('area_code').notNull(),
+		areaName: text('area_name'),
+		/** Persons entitled to vote (kiesgerechtigden). */
+		electorate: bigint('electorate', { mode: 'number' }),
+		/** Ballots cast / counted (opkomst). */
+		ballots: bigint('ballots', { mode: 'number' }),
+		valid: bigint('valid', { mode: 'number' }),
+		blank: bigint('blank', { mode: 'number' }),
+		invalid: bigint('invalid', { mode: 'number' })
+	},
+	(t) => [primaryKey({ columns: [t.electionId, t.level, t.areaCode] })]
+);
+
+// ── Step 3: world opinions (§4, A15) ─────────────────────────────────────────────────────
+// The /projects/world-opinions guide reads these. Curated rows (seeded from the former
+// src/lib/world-opinions/data.ts) keep Stan's text exactly; collectors add CHES / V-Party data.
+
+export const statsWoCountry = statsSchema.table('wo_country', {
+	iso3: char('iso3', { length: 3 }).primaryKey(),
+	/** ISO 3166-1 alpha-2, lower case (the guide's own id). */
+	id: text('id').notNull().unique(),
+	/** ISO 3166-1 numeric as a zero-padded string (world-atlas shape id). */
+	isoNum: text('iso_num').notNull(),
+	name: text('name').notNull(),
+	/** One of the 11 RegionId keys in src/lib/world-opinions/data.ts. */
+	region: text('region').notNull(),
+	/** left | market | western */
+	liberalMeaning: text('liberal_meaning'),
+	/** economy | culture | history | none */
+	mainDivide: text('main_divide'),
+	note: text('note'),
+	/** Country profile {center, fights[], timeline[{year,label}], blindSpots[]}, or null. */
+	profile: jsonb('profile'),
+	sort: integer('sort').notNull()
+});
+
+/** Inglehart–Welzel map positions; the guide's hand-placed values are wave 'curated'. */
+export const statsWoCulturalMap = statsSchema.table(
+	'wo_cultural_map',
+	{
+		iso3: char('iso3', { length: 3 }).notNull(),
+		wave: text('wave').notNull(),
+		yearFrom: integer('year_from'),
+		yearTo: integer('year_to'),
+		/** y axis: traditional (-) ↔ secular-rational (+). */
+		tradSecular: doublePrecision('trad_secular'),
+		/** x axis: survival (-) ↔ self-expression (+). */
+		survSelfexp: doublePrecision('surv_selfexp'),
+		n: integer('n'),
+		source: text('source')
+	},
+	(t) => [primaryKey({ columns: [t.iso3, t.wave] })]
+);
+
+/** WVS/EVS/Pew item aggregates (Tier C manual loads; empty until then). */
+export const statsWoValueScore = statsSchema.table(
+	'wo_value_score',
+	{
+		iso3: char('iso3', { length: 3 }).notNull(),
+		wave: text('wave').notNull(),
+		itemCode: text('item_code').notNull(),
+		itemLabel: text('item_label'),
+		stat: text('stat').notNull(),
+		value: doublePrecision('value'),
+		n: integer('n')
+	},
+	(t) => [primaryKey({ columns: [t.iso3, t.wave, t.itemCode, t.stat] })]
+);
+
+export const statsWoNarrative = statsSchema.table('wo_narrative', {
+	sort: integer('sort').primaryKey(),
+	event: text('event').notNull(),
+	years: text('years').notNull(),
+	/** [{country (alpha-2), story}] in display order. */
+	views: jsonb('views').notNull()
+});
+
+export const statsWoSource = statsSchema.table('wo_source', {
+	sort: integer('sort').primaryKey(),
+	name: text('name').notNull(),
+	href: text('href').notNull()
+});
+
+/**
+ * Parties. Curated guide parties: party_id `wo:<cc>:<short>`; data-driven parties:
+ * `pf:<partyfacts_id>` (or `ches:<id>` / `vp:<id>` when Party Facts has no link).
+ */
+export const statsWoParty = statsSchema.table(
+	'wo_party',
+	{
+		partyId: text('party_id').primaryKey(),
+		iso3: char('iso3', { length: 3 }),
+		name: text('name'),
+		abbrev: text('abbrev'),
+		family: text('family'),
+		chesId: integer('ches_id'),
+		manifestoId: integer('manifesto_id'),
+		vpartyId: integer('vparty_id'),
+		partyfactsId: integer('partyfacts_id'),
+		/** Display order of curated parties (null for data-driven rows). */
+		sort: integer('sort')
+	},
+	(t) => [index('wo_party_iso3_idx').on(t.iso3)]
+);
+
+export const statsWoPartyPosition = statsSchema.table(
+	'wo_party_position',
+	{
+		partyId: text('party_id').notNull(),
+		year: integer('year').notNull(),
+		/** ches, vparty */
+		source: text('source').notNull(),
+		/** lrgen, lrecon, galtan, eu_position, v2xpa_popul, v2pariglef, … */
+		dim: text('dim').notNull(),
+		value: doublePrecision('value'),
+		voteShare: doublePrecision('vote_share')
+	},
+	(t) => [primaryKey({ columns: [t.partyId, t.year, t.source, t.dim] })]
+);
+
+/** Stan's curated shared-compass placements (x = economic 0–10, y = social 0–10). */
+export const statsWoCompassOverride = statsSchema.table('wo_compass_override', {
+	partyId: text('party_id').primaryKey(),
+	x: doublePrecision('x').notNull(),
+	y: doublePrecision('y').notNull(),
+	note: text('note')
+});
+
+// ── Step 3: opinion feeds (A16) ──────────────────────────────────────────────────────────
+
+/** Raw Wikipedia wikitext of the poll pages, one row per revision seen. */
+export const statsOpWikiSnapshot = statsSchema.table(
+	'op_wiki_snapshot',
+	{
+		page: text('page').notNull(),
+		revid: bigint('revid', { mode: 'number' }).notNull(),
+		fetchedAt: timestamptz('fetched_at').notNull().defaultNow(),
+		wikitext: text('wikitext').notNull()
+	},
+	(t) => [primaryKey({ columns: [t.page, t.revid] })]
+);
+
+/** Parsed poll rows (long format): one value per poll × option. */
+export const statsOpPoll = statsSchema.table(
+	'op_poll',
+	{
+		/** nl-tk (seats), us-approval, us-approval-avg, us-generic, kr-approval, kr-party. */
+		series: text('series').notNull(),
+		/** `${fieldwork_end}|${pollster}|${n}` (n = order within that pollster and date). */
+		pollKey: text('poll_key').notNull(),
+		fieldworkEnd: date('fieldwork_end', { mode: 'string' }).notNull(),
+		pollster: text('pollster').notNull(),
+		sample: integer('sample'),
+		option: text('option').notNull(),
+		value: doublePrecision('value'),
+		seenAt: timestamptz('seen_at').notNull().defaultNow()
+	},
+	(t) => [
+		primaryKey({ columns: [t.series, t.pollKey, t.option] }),
+		index('op_poll_series_date_idx').on(t.series, t.fieldworkEnd)
+	]
+);
+
+/** Standard Eurobarometer volume A: most-important-issue and trust questions per country. */
+export const statsEbItem = statsSchema.table(
+	'eb_item',
+	{
+		/** 104.1 etc. */
+		wave: text('wave').notNull(),
+		fieldworkEnd: date('fieldwork_end', { mode: 'string' }),
+		/** mii_country, mii_personal, mii_eu, trust */
+		topic: text('topic').notNull(),
+		/** Sheet name in the wave's file (QA3, QA6_9, …). */
+		qcode: text('qcode').notNull(),
+		/** Answer (issue) for MII; institution for trust. */
+		item: text('item').notNull(),
+		/** Eurobarometer country code (EU27, BE, EL, …). */
+		country: text('country').notNull(),
+		/** Share of respondents, 0–1. */
+		share: doublePrecision('share')
+	},
+	(t) => [primaryKey({ columns: [t.wave, t.topic, t.item, t.country] })]
+);
+
+/** World Happiness Report: life evaluation (3-year average) and its explained parts. */
+export const statsWhrScore = statsSchema.table(
+	'whr_score',
+	{
+		year: integer('year').notNull(),
+		country: text('country').notNull(),
+		iso3: char('iso3', { length: 3 }),
+		rank: integer('rank'),
+		ladder: doublePrecision('ladder'),
+		lower: doublePrecision('lower'),
+		upper: doublePrecision('upper'),
+		gdp: doublePrecision('gdp'),
+		social: doublePrecision('social'),
+		health: doublePrecision('health'),
+		freedom: doublePrecision('freedom'),
+		generosity: doublePrecision('generosity'),
+		corruption: doublePrecision('corruption'),
+		dystopia: doublePrecision('dystopia')
+	},
+	(t) => [primaryKey({ columns: [t.year, t.country] })]
+);
+
+/** Google Trends daily trending searches (RSS snapshot). */
+export const statsGtTrending = statsSchema.table(
+	'gt_trending',
+	{
+		geo: text('geo').notNull(),
+		date: date('date', { mode: 'string' }).notNull(),
+		rank: integer('rank').notNull(),
+		query: text('query').notNull(),
+		approxTraffic: integer('approx_traffic'),
+		pubDate: timestamptz('pub_date'),
+		/** First related news headline + source, for context. */
+		news: text('news')
+	},
+	(t) => [primaryKey({ columns: [t.geo, t.date, t.query] })]
+);
+
+/** GDELT DOC timelinevol: share of monitored news coverage per theme and source country. */
+export const statsGdeltThemeVol = statsSchema.table(
+	'gdelt_theme_vol',
+	{
+		date: date('date', { mode: 'string' }).notNull(),
+		/** ISO 3166-1 alpha-3 of the source country. */
+		country: char('country', { length: 3 }).notNull(),
+		theme: text('theme').notNull(),
+		/** Percentage of all monitored articles (daily mean of GDELT's volume intensity). */
+		volPct: doublePrecision('vol_pct')
+	},
+	(t) => [primaryKey({ columns: [t.date, t.country, t.theme] })]
+);

src/lib/server/world-opinions.ts

@@ -0,0 +1,76 @@
+import { client } from '#lib/server/db/index.js';
+import { cached } from '#lib/server/stats.js';
+import type {
+	Country,
+	Divide,
+	LiberalMeaning,
+	Narrative,
+	Party,
+	Profile,
+	RegionId,
+	Source,
+	WorldOpinionsData
+} from '#lib/world-opinions/data.js';
+
+// /projects/world-opinions content from stats.wo_* (seeded from the former data.ts). Rebuilds
+// the exact shapes the components used, leaving optional keys out when the column is null.
+export function loadWorldOpinions(): Promise<WorldOpinionsData> {
+	return cached('world-opinions', async () => {
+		const [countryRows, partyRows, narrativeRows, sourceRows] = await Promise.all([
+			client`
+				select c.id, c.iso_num, c.name, c.region, c.liberal_meaning, c.main_divide, c.note,
+					c.profile, m.surv_selfexp, m.trad_secular
+				from stats.wo_country c
+				left join stats.wo_cultural_map m on m.iso3 = c.iso3 and m.wave = 'curated'
+				order by c.sort`,
+			client`
+				select c.id as country, p.abbrev, p.name, o.x, o.y, o.note
+				from stats.wo_party p
+				join stats.wo_compass_override o using (party_id)
+				join stats.wo_country c on c.iso3 = p.iso3
+				where p.sort is not null
+				order by p.sort`,
+			client`select event, years, views from stats.wo_narrative order by sort`,
+			client`select name, href from stats.wo_source order by sort`
+		]);
+		const countries: Country[] = [];
+		const profiles: Profile[] = [];
+		for (const r of countryRows) {
+			const country: Country = {
+				id: r.id,
+				iso: r.iso_num,
+				name: r.name,
+				region: r.region as RegionId
+			};
+			if (r.surv_selfexp != null && r.trad_secular != null)
+				country.values = { x: r.surv_selfexp, y: r.trad_secular };
+			if (r.liberal_meaning) country.liberal = r.liberal_meaning as LiberalMeaning;
+			if (r.main_divide) country.divide = r.main_divide as Divide;
+			if (r.note) country.note = r.note;
+			countries.push(country);
+			if (r.profile) profiles.push({ id: r.id, ...(r.profile as Omit<Profile, 'id'>) });
+		}
+		const parties: Party[] = partyRows.map((r) => {
+			const party: Party = {
+				country: r.country,
+				short: r.abbrev,
+				name: r.name,
+				econ: r.x,
+				social: r.y
+			};
+			if (r.note) party.note = r.note;
+			return party;
+		});
+		return {
+			countries,
+			parties,
+			profiles,
+			narratives: narrativeRows.map((r) => ({
+				event: r.event,
+				years: r.years,
+				views: r.views as Narrative['views']
+			})),
+			sources: sourceRows.map((r) => ({ name: r.name, href: r.href }) as Source)
+		};
+	});
+}

src/lib/world-opinions/CountryPanel.svelte

@@ -1,7 +1,9 @@
 <script lang="ts">
 	import EyeOff from 'lucide-svelte/icons/eye-off';
 	import Swords from 'lucide-svelte/icons/swords';
-	import { countryById, divides, liberalMeanings, profileById, regions } from './data';
+	import { divides, liberalMeanings, regions, worldOpinions } from './data';
+
+	const { countryById, profileById } = worldOpinions();
 
 	let { id }: { id: string } = $props();
 

src/lib/world-opinions/CultureMap.svelte

@@ -1,7 +1,9 @@
 <script lang="ts">
-	import { countries, regions, type RegionId } from './data';
+	import { regions, worldOpinions, type RegionId } from './data';
 	import { Button } from '#lib/components/ui/button/index.js';
 
+	const { countries } = worldOpinions();
+
 	const culturalMap = countries.flatMap((country) =>
 		country.values ? [{ ...country, x: country.values.x, y: country.values.y }] : []
 	);

src/lib/world-opinions/PartyCompass.svelte

@@ -1,7 +1,9 @@
 <script lang="ts">
-	import { countryById, parties, partyCountries } from './data';
+	import { worldOpinions } from './data';
 	import { Button } from '#lib/components/ui/button/index.js';
 
+	const { countryById, parties, partyCountries } = worldOpinions();
+
 	let { focus = null }: { focus?: string | null } = $props();
 
 	const MAX = 3;

src/lib/world-opinions/WorldMap.svelte

@@ -31,7 +31,9 @@
 </script>
 
 <script lang="ts">
-	import { countries, divides, liberalMeanings, type Country } from './data';
+	import { divides, liberalMeanings, worldOpinions, type Country } from './data';
+
+	const { countries } = worldOpinions();
 
 	let {
 		mode,

src/lib/world-opinions/data.test.ts

@@ -1,8 +1,14 @@
 import { describe, expect, it } from 'vitest';
 import world from 'world-atlas/countries-110m.json';
-import { countries, countryById, narratives, parties, partyCountries, profiles } from './data';
+import seed from '../../../scripts/stats/world-opinions-seed.json';
+import { buildWorldOpinions, type WorldOpinionsData } from './data';
 
-describe('world opinions data', () => {
+// The guide's content lives in stats.wo_*; this checks the snapshot the stats_wo_seed migration
+// was generated from (scripts/stats/world-opinions-seed.json).
+const { countries, countryById, narratives, parties, partyCountries, profiles } =
+	buildWorldOpinions(seed as WorldOpinionsData);
+
+describe('world opinions seed', () => {
 	it('has unique country ids and ISO codes', () => {
 		expect(new Set(countries.map((country) => country.id)).size).toBe(countries.length);
 		expect(new Set(countries.map((country) => country.iso)).size).toBe(countries.length);

src/lib/world-opinions/data.ts

@@ -1,6 +1,10 @@
 // World Opinions: a hand-curated orientation guide, not a dataset of record.
 // Positions are approximate and meant for comparing countries/parties relative
-// to each other. Sources are listed in `sources` at the bottom of this file.
+// to each other. The curated content (countries, party placements, profiles,
+// narratives, sources) lives in the `stats.wo_*` tables (seeded by the
+// stats_wo_seed migration) and is loaded by +page.server.ts; this file keeps
+// the types, the label dictionaries and the lookups the components share.
+import { getContext, setContext } from 'svelte';
 
 export type RegionId =
 	| 'north-america'
@@ -63,483 +67,6 @@ export type Country = {
 	note?: string;
 };
 
-export const countries: Country[] = [
-	{
-		id: 'se',
-		iso: '752',
-		name: 'Sweden',
-		region: 'nordics',
-		values: { x: 2.4, y: 1.8 },
-		liberal: 'market',
-		divide: 'economy'
-	},
-	{
-		id: 'dk',
-		iso: '208',
-		name: 'Denmark',
-		region: 'nordics',
-		values: { x: 2.3, y: 1.2 },
-		liberal: 'market',
-		divide: 'economy',
-		note: '"Venstre" (literally "Left") is the liberal, center-right party'
-	},
-	{
-		id: 'no',
-		iso: '578',
-		name: 'Norway',
-		region: 'nordics',
-		values: { x: 2.1, y: 1.4 },
-		liberal: 'market',
-		divide: 'economy'
-	},
-	{
-		id: 'fi',
-		iso: '246',
-		name: 'Finland',
-		region: 'nordics',
-		values: { x: 1.6, y: 1.0 },
-		liberal: 'market',
-		divide: 'economy'
-	},
-	{
-		id: 'nl',
-		iso: '528',
-		name: 'Netherlands',
-		region: 'western-europe',
-		values: { x: 2.0, y: 1.1 },
-		liberal: 'market',
-		divide: 'economy',
-		note: 'Liberal VVD is the main center-right party'
-	},
-	{
-		id: 'be',
-		iso: '056',
-		name: 'Belgium',
-		region: 'western-europe',
-		values: { x: 1.4, y: 0.8 },
-		liberal: 'market',
-		divide: 'economy',
-		note: 'Language (Flemish vs French-speaking) splits every party family'
-	},
-	{
-		id: 'de',
-		iso: '276',
-		name: 'Germany',
-		region: 'western-europe',
-		values: { x: 1.3, y: 1.4 },
-		liberal: 'market',
-		divide: 'economy',
-		note: 'Liberal FDP = pro-market, low taxes'
-	},
-	{
-		id: 'at',
-		iso: '040',
-		name: 'Austria',
-		region: 'western-europe',
-		values: { x: 1.3, y: 0.6 },
-		liberal: 'market',
-		divide: 'economy'
-	},
-	{
-		id: 'ch',
-		iso: '756',
-		name: 'Switzerland',
-		region: 'western-europe',
-		values: { x: 1.9, y: 0.7 },
-		liberal: 'market',
-		divide: 'economy'
-	},
-	{
-		id: 'gb',
-		iso: '826',
-		name: 'United Kingdom',
-		region: 'western-europe',
-		values: { x: 1.8, y: 0.3 },
-		liberal: 'market',
-		divide: 'economy',
-		note: 'Liberal Democrats sit in the center; Brexit cut across old lines'
-	},
-	{
-		id: 'ie',
-		iso: '372',
-		name: 'Ireland',
-		region: 'western-europe',
-		values: { x: 1.4, y: -0.4 },
-		liberal: 'market',
-		divide: 'history',
-		note: 'The two big parties descend from opposite sides of the 1922 civil war'
-	},
-	{
-		id: 'fr',
-		iso: '250',
-		name: 'France',
-		region: 'western-europe',
-		values: { x: 1.1, y: 0.8 },
-		liberal: 'market',
-		divide: 'economy',
-		note: '"Libéral" means pro-market, often used as a criticism'
-	},
-	{
-		id: 'es',
-		iso: '724',
-		name: 'Spain',
-		region: 'western-europe',
-		values: { x: 1.3, y: 0.3 },
-		liberal: 'market',
-		divide: 'economy',
-		note: 'Regional nationalism (Catalonia, Basque Country) is a second axis'
-	},
-	{
-		id: 'pt',
-		iso: '620',
-		name: 'Portugal',
-		region: 'western-europe',
-		values: { x: 0.8, y: -0.3 },
-		liberal: 'market',
-		divide: 'economy'
-	},
-	{
-		id: 'it',
-		iso: '380',
-		name: 'Italy',
-		region: 'western-europe',
-		values: { x: 0.5, y: 0.0 },
-		liberal: 'market',
-		divide: 'economy'
-	},
-	{
-		id: 'gr',
-		iso: '300',
-		name: 'Greece',
-		region: 'eastern-europe',
-		values: { x: 0.2, y: 0.0 },
-		liberal: 'market',
-		divide: 'economy'
-	},
-	{
-		id: 'us',
-		iso: '840',
-		name: 'United States',
-		region: 'north-america',
-		values: { x: 1.3, y: -0.5 },
-		liberal: 'left',
-		divide: 'culture',
-		note: '"Liberal" = Democrat; the Democrats sit near Europe\'s center economically'
-	},
-	{
-		id: 'ca',
-		iso: '124',
-		name: 'Canada',
-		region: 'north-america',
-		values: { x: 1.8, y: 0.0 },
-		liberal: 'left',
-		divide: 'economy'
-	},
-	{
-		id: 'mx',
-		iso: '484',
-		name: 'Mexico',
-		region: 'latin-america',
-		values: { x: 0.6, y: -1.3 },
-		liberal: 'market',
-		divide: 'economy'
-	},
-	{
-		id: 'br',
-		iso: '076',
-		name: 'Brazil',
-		region: 'latin-america',
-		values: { x: 0.1, y: -1.0 },
-		liberal: 'market',
-		divide: 'economy',
-		note: 'Bolsonaro\'s party is literally called "Partido Liberal"'
-	},
-	{
-		id: 'ar',
-		iso: '032',
-		name: 'Argentina',
-		region: 'latin-america',
-		values: { x: 0.7, y: -0.6 },
-		liberal: 'market',
-		divide: 'economy',
-		note: 'President Milei calls himself a "liberal-libertarian"'
-	},
-	{
-		id: 'cl',
-		iso: '152',
-		name: 'Chile',
-		region: 'latin-america',
-		values: { x: 0.8, y: -0.3 },
-		liberal: 'market',
-		divide: 'economy'
-	},
-	{
-		id: 'co',
-		iso: '170',
-		name: 'Colombia',
-		region: 'latin-america',
-		values: { x: 0.5, y: -1.6 },
-		divide: 'economy'
-	},
-	{
-		id: 'pe',
-		iso: '604',
-		name: 'Peru',
-		region: 'latin-america',
-		values: { x: 0.3, y: -1.1 },
-		liberal: 'market',
-		divide: 'economy'
-	},
-	{
-		id: 've',
-		iso: '862',
-		name: 'Venezuela',
-		region: 'latin-america',
-		values: { x: 0.4, y: -1.5 },
-		divide: 'none'
-	},
-	{
-		id: 'au',
-		iso: '036',
-		name: 'Australia',
-		region: 'oceania',
-		values: { x: 2.0, y: 0.2 },
-		liberal: 'market',
-		divide: 'economy',
-		note: 'The Liberal Party is the main conservative party'
-	},
-	{
-		id: 'nz',
-		iso: '554',
-		name: 'New Zealand',
-		region: 'oceania',
-		values: { x: 2.2, y: 0.1 },
-		liberal: 'market',
-		divide: 'economy'
-	},
-	{
-		id: 'pl',
-		iso: '616',
-		name: 'Poland',
-		region: 'eastern-europe',
-		values: { x: 0.2, y: -0.6 },
-		liberal: 'western',
-		divide: 'culture'
-	},
-	{
-		id: 'cz',
-		iso: '203',
-		name: 'Czechia',
-		region: 'eastern-europe',
-		values: { x: 0.5, y: 1.2 },
-		liberal: 'western',
-		divide: 'culture'
-	},
-	{
-		id: 'hu',
-		iso: '348',
-		name: 'Hungary',
-		region: 'eastern-europe',
-		values: { x: -0.4, y: 0.4 },
-		liberal: 'western',
-		divide: 'culture',
-		note: 'Orbán calls his model "illiberal democracy"'
-	},
-	{
-		id: 'ro',
-		iso: '642',
-		name: 'Romania',
-		region: 'eastern-europe',
-		values: { x: -0.4, y: -0.6 },
-		liberal: 'western',
-		divide: 'culture'
-	},
-	{
-		id: 'rs',
-		iso: '688',
-		name: 'Serbia',
-		region: 'eastern-europe',
-		values: { x: -0.6, y: 0.1 },
-		liberal: 'western',
-		divide: 'history'
-	},
-	{
-		id: 'ua',
-		iso: '804',
-		name: 'Ukraine',
-		region: 'eastern-europe',
-		values: { x: -0.9, y: 0.2 },
-		divide: 'history',
-		note: 'Politics long split between pro-Western and pro-Russian orientation'
-	},
-	{
-		id: 'ru',
-		iso: '643',
-		name: 'Russia',
-		region: 'eastern-europe',
-		values: { x: -1.0, y: 0.6 },
-		liberal: 'western',
-		divide: 'none',
-		note: '"Liberal" = the pro-Western opposition, mostly in exile or prison'
-	},
-	{
-		id: 'kz',
-		iso: '398',
-		name: 'Kazakhstan',
-		region: 'eastern-europe',
-		values: { x: -0.6, y: 0.4 },
-		divide: 'none'
-	},
-	{
-		id: 'tr',
-		iso: '792',
-		name: 'Turkey',
-		region: 'middle-east',
-		values: { x: -0.6, y: -0.9 },
-		liberal: 'western',
-		divide: 'culture',
-		note: 'Main divide: religious-conservative vs secular'
-	},
-	{
-		id: 'ir',
-		iso: '364',
-		name: 'Iran',
-		region: 'middle-east',
-		values: { x: -0.9, y: -0.6 },
-		liberal: 'western',
-		divide: 'none'
-	},
-	{
-		id: 'iq',
-		iso: '368',
-		name: 'Iraq',
-		region: 'middle-east',
-		values: { x: -1.2, y: -1.3 },
-		divide: 'culture'
-	},
-	{
-		id: 'eg',
-		iso: '818',
-		name: 'Egypt',
-		region: 'middle-east',
-		values: { x: -1.5, y: -1.5 },
-		divide: 'none'
-	},
-	{ id: 'ma', iso: '504', name: 'Morocco', region: 'middle-east', values: { x: -1.2, y: -1.5 } },
-	{ id: 'sa', iso: '682', name: 'Saudi Arabia', region: 'middle-east', divide: 'none' },
-	{ id: 'ng', iso: '566', name: 'Nigeria', region: 'africa', values: { x: -0.3, y: -1.8 } },
-	{ id: 'et', iso: '231', name: 'Ethiopia', region: 'africa', values: { x: -0.9, y: -1.5 } },
-	{ id: 'ke', iso: '404', name: 'Kenya', region: 'africa', values: { x: -0.4, y: -1.5 } },
-	{
-		id: 'za',
-		iso: '710',
-		name: 'South Africa',
-		region: 'africa',
-		values: { x: 0.0, y: -1.2 },
-		divide: 'history',
-		note: 'Politics still shaped by apartheid and the liberation struggle'
-	},
-	{
-		id: 'in',
-		iso: '356',
-		name: 'India',
-		region: 'south-asia',
-		values: { x: -0.5, y: -0.5 },
-		divide: 'culture',
-		note: 'Main divide: Hindu nationalism vs secular pluralism'
-	},
-	{ id: 'pk', iso: '586', name: 'Pakistan', region: 'south-asia', values: { x: -1.0, y: -1.4 } },
-	{
-		id: 'bd',
-		iso: '050',
-		name: 'Bangladesh',
-		region: 'south-asia',
-		values: { x: -1.0, y: -1.1 },
-		divide: 'history'
-	},
-	{
-		id: 'cn',
-		iso: '156',
-		name: 'China',
-		region: 'east-asia',
-		values: { x: -0.9, y: 1.5 },
-		liberal: 'western',
-		divide: 'none',
-		note: '"Liberal" = pro-Western intellectuals, a label the state treats with suspicion'
-	},
-	{
-		id: 'jp',
-		iso: '392',
-		name: 'Japan',
-		region: 'east-asia',
-		values: { x: 0.4, y: 1.8 },
-		liberal: 'market',
-		divide: 'history',
-		note: 'The "Liberal Democratic Party" is the conservative party in power since 1955'
-	},
-	{
-		id: 'kr',
-		iso: '410',
-		name: 'South Korea',
-		region: 'east-asia',
-		values: { x: -0.3, y: 1.2 },
-		liberal: 'left',
-		divide: 'history',
-		note: 'Divide is mostly about North Korea, Japan and the US'
-	},
-	{
-		id: 'tw',
-		iso: '158',
-		name: 'Taiwan',
-		region: 'east-asia',
-		values: { x: 0.0, y: 1.3 },
-		divide: 'history',
-		note: 'Main divide: how close to be with mainland China'
-	},
-	{
-		id: 'vn',
-		iso: '704',
-		name: 'Vietnam',
-		region: 'southeast-asia',
-		values: { x: -0.6, y: 0.5 },
-		divide: 'none'
-	},
-	{
-		id: 'th',
-		iso: '764',
-		name: 'Thailand',
-		region: 'southeast-asia',
-		values: { x: 0.0, y: -0.4 },
-		divide: 'history',
-		note: 'Main divide: monarchy and military vs reformists'
-	},
-	{
-		id: 'my',
-		iso: '458',
-		name: 'Malaysia',
-		region: 'southeast-asia',
-		values: { x: -0.6, y: -0.9 },
-		divide: 'culture'
-	},
-	{
-		id: 'ph',
-		iso: '608',
-		name: 'Philippines',
-		region: 'southeast-asia',
-		values: { x: -0.2, y: -1.4 }
-	},
-	{
-		id: 'id',
-		iso: '360',
-		name: 'Indonesia',
-		region: 'southeast-asia',
-		values: { x: -0.8, y: -1.2 },
-		divide: 'culture'
-	}
-];
-
-export const countryById = new Map(countries.map((country) => [country.id, country]));
-
 // ---------------------------------------------------------------------------
 // Party compass: parties on one shared scale
 // econ: 0 = far left (state-led, redistributive) … 10 = far right (free market)
@@ -557,154 +84,7 @@ export type Party = {
 	note?: string;
 };
 
-export const partyCountries = [
+// Country profiles, kept short enough to scan.
-	'us',
-	'nl',
-	'de',
-	'gb',
-	'fr',
-	'se',
-	'pl',
-	'jp',
-	'kr',
-	'in',
-	'br',
-	'ca'
-];
-
-export const parties: Party[] = [
-	{
-		country: 'us',
-		short: 'Dem',
-		name: 'Democratic Party',
-		econ: 4.8,
-		social: 3.5,
-		note: 'The US "liberals": near Europe\'s center economically'
-	},
-	{ country: 'us', short: 'GOP', name: 'Republican Party', econ: 7.8, social: 8.0 },
-
-	{ country: 'nl', short: 'GL-PvdA', name: 'GroenLinks-PvdA', econ: 2.5, social: 1.5 },
-	{ country: 'nl', short: 'SP', name: 'Socialistische Partij', econ: 1.5, social: 5.0 },
-	{ country: 'nl', short: 'D66', name: 'Democraten 66', econ: 5.5, social: 1.5 },
-	{ country: 'nl', short: 'CDA', name: 'Christen-Democratisch Appèl', econ: 6.0, social: 6.0 },
-	{
-		country: 'nl',
-		short: 'VVD',
-		name: 'Volkspartij voor Vrijheid en Democratie',
-		econ: 7.5,
-		social: 5.0,
-		note: 'Dutch "liberals": market-liberal, center-right'
-	},
-	{ country: 'nl', short: 'BBB', name: 'BoerBurgerBeweging', econ: 6.0, social: 7.5 },
-	{
-		country: 'nl',
-		short: 'PVV',
-		name: 'Partij voor de Vrijheid',
-		econ: 4.5,
-		social: 9.0,
-		note: 'Radical right on immigration, centrist on welfare'
-	},
-
-	{ country: 'de', short: 'Linke', name: 'Die Linke', econ: 1.0, social: 2.5 },
-	{ country: 'de', short: 'Grüne', name: 'Bündnis 90/Die Grünen', econ: 3.0, social: 1.5 },
-	{ country: 'de', short: 'SPD', name: 'Sozialdemokratische Partei', econ: 3.5, social: 3.5 },
-	{ country: 'de', short: 'BSW', name: 'Bündnis Sahra Wagenknecht', econ: 2.0, social: 7.0 },
-	{
-		country: 'de',
-		short: 'FDP',
-		name: 'Freie Demokratische Partei',
-		econ: 8.5,
-		social: 3.0,
-		note: 'German "liberals": socially open, strongly pro-market'
-	},
-	{ country: 'de', short: 'CDU', name: 'CDU/CSU', econ: 6.5, social: 6.0 },
-	{ country: 'de', short: 'AfD', name: 'Alternative für Deutschland', econ: 7.0, social: 9.5 },
-
-	{ country: 'gb', short: 'Green', name: 'Green Party', econ: 1.5, social: 1.5 },
-	{ country: 'gb', short: 'Lab', name: 'Labour', econ: 3.8, social: 3.5 },
-	{ country: 'gb', short: 'LD', name: 'Liberal Democrats', econ: 5.0, social: 2.5 },
-	{ country: 'gb', short: 'Con', name: 'Conservatives', econ: 7.0, social: 6.5 },
-	{ country: 'gb', short: 'Reform', name: 'Reform UK', econ: 7.5, social: 8.5 },
-
-	{ country: 'fr', short: 'LFI', name: 'La France Insoumise', econ: 1.0, social: 2.5 },
-	{ country: 'fr', short: 'PS', name: 'Parti Socialiste', econ: 3.0, social: 3.0 },
-	{ country: 'fr', short: 'RE', name: 'Renaissance', econ: 6.5, social: 4.0 },
-	{ country: 'fr', short: 'LR', name: 'Les Républicains', econ: 7.5, social: 7.0 },
-	{
-		country: 'fr',
-		short: 'RN',
-		name: 'Rassemblement National',
-		econ: 4.5,
-		social: 9.0,
-		note: 'Economically left of Macron: lower retirement age, protectionism'
-	},
-
-	{ country: 'se', short: 'V', name: 'Vänsterpartiet', econ: 1.0, social: 2.0 },
-	{ country: 'se', short: 'S', name: 'Socialdemokraterna', econ: 3.0, social: 3.5 },
-	{ country: 'se', short: 'C', name: 'Centerpartiet', econ: 7.0, social: 2.5 },
-	{ country: 'se', short: 'M', name: 'Moderaterna', econ: 7.5, social: 5.5 },
-	{ country: 'se', short: 'SD', name: 'Sverigedemokraterna', econ: 5.5, social: 9.0 },
-
-	{ country: 'pl', short: 'Lewica', name: 'Lewica', econ: 2.0, social: 2.0 },
-	{ country: 'pl', short: 'KO', name: 'Koalicja Obywatelska', econ: 6.5, social: 4.0 },
-	{
-		country: 'pl',
-		short: 'PiS',
-		name: 'Prawo i Sprawiedliwość',
-		econ: 3.5,
-		social: 9.0,
-		note: 'Culturally right, economically left (big family benefits)'
-	},
-	{ country: 'pl', short: 'Konf', name: 'Konfederacja', econ: 9.0, social: 9.0 },
-
-	{ country: 'jp', short: 'JCP', name: 'Japanese Communist Party', econ: 1.0, social: 3.0 },
-	{ country: 'jp', short: 'CDP', name: 'Constitutional Democratic Party', econ: 4.0, social: 3.5 },
-	{ country: 'jp', short: 'Komeito', name: 'Komeito', econ: 5.0, social: 5.5 },
-	{
-		country: 'jp',
-		short: 'LDP',
-		name: 'Liberal Democratic Party',
-		econ: 6.0,
-		social: 7.0,
-		note: 'Called "Liberal", governs as conservatives since 1955'
-	},
-	{ country: 'jp', short: 'Ishin', name: 'Nippon Ishin no Kai', econ: 7.5, social: 6.0 },
-	{ country: 'jp', short: 'Sansei', name: 'Sanseitō', econ: 6.0, social: 9.0 },
-
-	{ country: 'kr', short: 'Justice', name: 'Justice Party', econ: 1.5, social: 1.5 },
-	{
-		country: 'kr',
-		short: 'DPK',
-		name: 'Democratic Party of Korea',
-		econ: 4.0,
-		social: 4.5,
-		note: 'The "progressive" camp, cautious on social issues'
-	},
-	{ country: 'kr', short: 'PPP', name: 'People Power Party', econ: 7.0, social: 7.0 },
-
-	{ country: 'in', short: 'INC', name: 'Indian National Congress', econ: 4.5, social: 4.0 },
-	{
-		country: 'in',
-		short: 'BJP',
-		name: 'Bharatiya Janata Party',
-		econ: 6.0,
-		social: 8.5,
-		note: 'Hindu-nationalist, pro-business, big welfare schemes'
-	},
-
-	{ country: 'br', short: 'PT', name: 'Partido dos Trabalhadores', econ: 2.5, social: 4.5 },
-	{ country: 'br', short: 'MDB', name: 'Movimento Democrático Brasileiro', econ: 6.0, social: 6.0 },
-	{ country: 'br', short: 'PL', name: 'Partido Liberal', econ: 7.5, social: 9.0 },
-
-	{ country: 'ca', short: 'NDP', name: 'New Democratic Party', econ: 2.5, social: 2.5 },
-	{ country: 'ca', short: 'LPC', name: 'Liberal Party of Canada', econ: 4.8, social: 3.0 },
-	{ country: 'ca', short: 'CPC', name: 'Conservative Party of Canada', econ: 7.5, social: 7.0 }
-];
-
-// ---------------------------------------------------------------------------
-// Country profiles, kept short enough to scan
-// ---------------------------------------------------------------------------
-
 export type Profile = {
 	id: string;
 	/** What "normal" politics looks like, in one line. */
@@ -714,263 +94,43 @@ export type Profile = {
 	blindSpots: string[];
 };
 
-export const profiles: Profile[] = [
+// One event, many stories.
-	{
-		id: 'us',
-		center: 'Individual liberty and suspicion of government, on both sides',
-		fights: ['abortion', 'guns', 'race', 'immigration', 'religion'],
-		timeline: [
-			{ year: '1776', label: 'revolt against a distant state' },
-			{ year: '1861', label: 'civil war over slavery' },
-			{ year: '1945', label: 'wins WWII, superpower' },
-			{ year: '1964', label: 'Civil Rights Act' },
-			{ year: '1991', label: 'wins the Cold War' }
-		],
-		blindSpots: [
-			'assumes its left/right is universal',
-			'partisan media bubbles',
-			'little view of other systems'
-		]
-	},
-	{
-		id: 'nl',
-		center: 'Coalition consensus ("polder model") around a strong welfare state',
-		fights: ['immigration', 'housing', 'nitrogen & farmers', 'EU'],
-		timeline: [
-			{ year: '1600s', label: 'Golden Age, trade empire' },
-			{ year: '1940', label: 'German occupation' },
-			{ year: '1949', label: 'loses Indonesia' },
-			{ year: '1960s', label: 'pillars break, secularization' },
-			{ year: '2002', label: 'Fortuyn, immigration debate' }
-		],
-		blindSpots: ['tolerance self-image vs discrimination', 'colonial past taught thinly']
-	},
-	{
-		id: 'de',
-		center: 'Social market economy, fiscal caution, "never again"',
-		fights: ['migration', 'energy & climate', 'East vs West', 'debt brake'],
-		timeline: [
-			{ year: '1923', label: 'hyperinflation' },
-			{ year: '1933', label: 'Nazi dictatorship' },
-			{ year: '1949', label: 'split + economic miracle' },
-			{ year: '1990', label: 'reunification' },
-			{ year: '2015', label: 'refugee crisis' }
-		],
-		blindSpots: ['debt as moral failing', 'Russian gas as "just trade"']
-	},
-	{
-		id: 'fr',
-		center: 'Strong central state, laïcité, protest as normal politics',
-		fights: ['pensions', 'Paris vs periphery', 'Islam & laïcité', 'EU'],
-		timeline: [
-			{ year: '1789', label: 'Revolution invents left/right' },
-			{ year: '1940', label: 'defeat and Vichy' },
-			{ year: '1962', label: 'loses Algerian war' },
-			{ year: '1968', label: "May '68" },
-			{ year: '2018', label: 'Yellow Vests' }
-		],
-		blindSpots: ['colorblind universalism hides racism', 'influence in former African colonies']
-	},
-	{
-		id: 'gb',
-		center: 'Pragmatic and market-friendly, but the NHS is sacred',
-		fights: ['Brexit', 'class', 'North vs South', 'Scotland'],
-		timeline: [
-			{ year: '1900', label: 'largest empire in history' },
-			{ year: '1940', label: '"stood alone"' },
-			{ year: '1948', label: 'NHS founded' },
-			{ year: '1979', label: 'Thatcher' },
-			{ year: '2016', label: 'Brexit vote' }
-		],
-		blindSpots: ['empire taught selectively', 'overrates the "special relationship"']
-	},
-	{
-		id: 'se',
-		center: 'High-trust, high-tax welfare state; very secular and individualist',
-		fights: ['immigration', 'gang violence', 'private welfare', 'NATO'],
-		timeline: [
-			{ year: '1814', label: 'last war fought' },
-			{ year: '1932', label: "Social Democrats' era" },
-			{ year: '1939', label: 'neutral in WWII' },
-			{ year: '2015', label: 'record asylum intake' },
-			{ year: '2024', label: 'joins NATO' }
-		],
-		blindSpots: ['consensus makes disagreement taboo', 'neutrality myth (iron to Germany)']
-	},
-	{
-		id: 'pl',
-		center: 'Pro-NATO and wary of Russia across all parties',
-		fights: ['Catholic vs liberal', 'abortion', 'rule of law', 'EU'],
-		timeline: [
-			{ year: '1795', label: 'erased by partition' },
-			{ year: '1939', label: 'invaded by Germany and USSR' },
-			{ year: '1980', label: 'Solidarity movement' },
-			{ year: '1989', label: 'communism falls' },
-			{ year: '2004', label: 'joins the EU' }
-		],
-		blindSpots: ['victimhood crowds out dark chapters', 'distrust of Germany']
-	},
-	{
-		id: 'ru',
-		center: 'Strong state and great-power status; opposition repressed',
-		fights: ['largely suppressed in public'],
-		timeline: [
-			{ year: '1812', label: 'Napoleon invades' },
-			{ year: '1941', label: 'Nazi invasion, ~27M dead' },
-			{ year: '1991', label: 'USSR collapses' },
-			{ year: '1990s', label: 'chaos and poverty' },
-			{ year: '2022', label: 'full invasion of Ukraine' }
-		],
-		blindSpots: [
-			'state-controlled media',
-			'neighbors seen as sphere of influence',
-			'Soviet crimes downplayed'
-		]
-	},
-	{
-		id: 'cn',
-		center: 'Order, growth and national rejuvenation, led by the Party',
-		fights: ['youth jobs', 'property crisis', 'local corruption'],
-		timeline: [
-			{ year: '1839', label: 'Opium Wars, "humiliation"' },
-			{ year: '1937', label: 'Japanese invasion' },
-			{ year: '1966', label: 'Cultural Revolution' },
-			{ year: '1978', label: 'reform and opening' },
-			{ year: '1989', label: 'Tiananmen (unspoken)' }
-		],
-		blindSpots: ['censorship shapes history', 'criticism read as new humiliation']
-	},
-	{
-		id: 'jp',
-		center: 'Stability and harmony; one party governing since 1955',
-		fights: ['pacifist constitution', 'aging & immigration', 'war memory'],
-		timeline: [
-			{ year: '1868', label: 'Meiji modernization' },
-			{ year: '1937', label: 'war in Asia' },
-			{ year: '1945', label: 'atomic bombs, defeat' },
-			{ year: '1947', label: 'pacifist constitution' },
-			{ year: '1990', label: 'bubble bursts' }
-		],
-		blindSpots: ['wartime atrocities taught thinly', 'sees itself as war victim']
-	},
-	{
-		id: 'kr',
-		center: 'Fierce democracy won by protest; hyper-competitive society',
-		fights: ['North Korea', 'Japan', 'gender', 'chaebol power'],
-		timeline: [
-			{ year: '1910', label: 'Japanese colonization' },
-			{ year: '1950', label: 'Korean War' },
-			{ year: '1980', label: 'Gwangju massacre' },
-			{ year: '1987', label: 'democracy' },
-			{ year: '2024', label: 'martial law crisis' }
-		],
-		blindSpots: ['US and Japan seen through party lens']
-	},
-	{
-		id: 'in',
-		center: 'Largest democracy; religion, caste and language drive politics',
-		fights: ['Hindu nationalism', 'caste quotas', 'language', 'center vs states'],
-		timeline: [
-			{ year: '1858', label: 'British Raj' },
-			{ year: '1947', label: 'independence and Partition' },
-			{ year: '1961', label: 'Non-Aligned Movement' },
-			{ year: '1991', label: 'economic liberalization' },
-			{ year: '2014', label: 'BJP majority' }
-		],
-		blindSpots: ['polarized media', 'reduced to poverty & religion by outsiders']
-	},
-	{
-		id: 'br',
-		center: 'Inequality makes redistribution central; religious society',
-		fights: ['Lula vs Bolsonaro', 'crime', 'Amazon vs agribusiness'],
-		timeline: [
-			{ year: '1888', label: 'last to abolish slavery' },
-			{ year: '1964', label: 'military coup' },
-			{ year: '1985', label: 'democracy returns' },
-			{ year: '1994', label: 'ends hyperinflation' },
-			{ year: '2014', label: 'Car Wash scandal' }
-		],
-		blindSpots: ['"racial democracy" myth', 'WhatsApp information bubbles']
-	},
-	{
-		id: 'tr',
-		center: 'National pride, strong state, balancing West and region',
-		fights: ['religious vs secular', 'Kurdish question', 'inflation'],
-		timeline: [
-			{ year: '1915', label: 'Armenian genocide (denied)' },
-			{ year: '1920', label: 'Sèvres, fear of partition' },
-			{ year: '1923', label: "Atatürk's secular republic" },
-			{ year: '1980', label: 'military coup' },
-			{ year: '2016', label: 'failed coup' }
-		],
-		blindSpots: ['genocide denial', 'restricted press', 'Western criticism read as hostility']
-	}
-];
-
-export const profileById = new Map(profiles.map((profile) => [profile.id, profile]));
-
-// ---------------------------------------------------------------------------
-// One event, many stories
-// ---------------------------------------------------------------------------
-
 export type Narrative = {
 	event: string;
 	years: string;
 	views: { country: string; story: string }[];
 };
 
-export const narratives: Narrative[] = [
+export type Source = { name: string; href: string };
-	{
-		event: 'World War II',
-		years: '1939–45',
-		views: [
-			{ country: 'us', story: 'D-Day: democracy beats fascism' },
-			{ country: 'ru', story: 'We broke Hitler, 27M dead' },
-			{ country: 'cn', story: 'Our war began in 1937' },
-			{ country: 'pl', story: 'Liberation became Soviet occupation' },
-			{ country: 'jp', story: 'Hiroshima: we suffered' }
-		]
-	},
-	{
-		event: 'End of the Cold War',
-		years: '1989–91',
-		views: [
-			{ country: 'us', story: 'We won: the end of history' },
-			{ country: 'pl', story: 'We freed ourselves' },
-			{ country: 'ru', story: 'Collapse and humiliation' },
-			{ country: 'cn', story: 'A warning: never lose control' }
-		]
-	},
-	{
-		event: 'Colonialism',
-		years: '1500s–1970s',
-		views: [
-			{ country: 'nl', story: 'Golden Age trade, now debated' },
-			{ country: 'gb', story: 'Empire, remembered fondly by many' },
-			{ country: 'in', story: 'Extraction and Partition' },
-			{ country: 'cn', story: 'Century of humiliation' },
-			{ country: 'br', story: 'Conquest, then US dominance' }
-		]
-	},
-	{
-		event: 'Iraq War',
-		years: '2003',
-		views: [
-			{ country: 'us', story: 'Disarmament, later a mistake' },
-			{ country: 'fr', story: 'We warned you' },
-			{ country: 'iq', story: 'Invasion, then civil war' },
-			{ country: 'ru', story: 'Proof the rules are selective' }
-		]
-	}
-];
 
-export const sources = [
+/** What the page loads: plain arrays in display order. */
-	{
+export type WorldOpinionsData = {
-		name: 'World Values Survey / EVS cultural map',
+	countries: Country[];
-		href: 'https://www.worldvaluessurvey.org/WVSContents.jsp?CMSID=Findings'
+	parties: Party[];
-	},
+	profiles: Profile[];
-	{ name: 'Chapel Hill Expert Survey', href: 'https://www.chesdata.eu/' },
+	narratives: Narrative[];
-	{ name: 'Manifesto Project', href: 'https://manifesto-project.wzb.eu/' },
+	sources: Source[];
-	{ name: 'Pew Research Center', href: 'https://www.pewresearch.org/global/' },
+};
-	{ name: 'Natural Earth (map shapes)', href: 'https://www.naturalearthdata.com/' }
+
-];
+/** The loaded data plus the lookups the components use. */
+export type WorldOpinions = WorldOpinionsData & {
+	countryById: Map<string, Country>;
+	profileById: Map<string, Profile>;
+	/** Countries with compass parties, in the order their parties are listed. */
+	partyCountries: string[];
+};
+
+export function buildWorldOpinions(data: WorldOpinionsData): WorldOpinions {
+	return {
+		...data,
+		countryById: new Map(data.countries.map((country) => [country.id, country])),
+		profileById: new Map(data.profiles.map((profile) => [profile.id, profile])),
+		partyCountries: [...new Set(data.parties.map((party) => party.country))]
+	};
+}
+
+const KEY = Symbol('world-opinions');
+
+/** Called once by the page; the map, panels and compass read it with `worldOpinions()`. */
+export const provideWorldOpinions = (data: WorldOpinions) => setContext(KEY, data);
+export const worldOpinions = () => getContext<WorldOpinions>(KEY);

src/routes/projects/world-opinions/+page.server.ts

@@ -0,0 +1,4 @@
+import { loadWorldOpinions } from '#lib/server/world-opinions.js';
+import type { PageServerLoad } from './$types';
+
+export const load: PageServerLoad = async () => ({ wo: await loadWorldOpinions() });

src/routes/projects/world-opinions/+page.svelte

@@ -5,9 +5,14 @@
 	import CultureMap from '#lib/world-opinions/CultureMap.svelte';
 	import PartyCompass from '#lib/world-opinions/PartyCompass.svelte';
 	import WorldMap, { type MapMode } from '#lib/world-opinions/WorldMap.svelte';
-	import { countries, countryById, narratives, sources } from '#lib/world-opinions/data.js';
+	import { buildWorldOpinions, provideWorldOpinions } from '#lib/world-opinions/data.js';
 	import { Button } from '#lib/components/ui/button/index.js';
 
+	let { data } = $props();
+	// Static per load (stats.wo_*), so it is read once and shared with the components.
+	const wo = provideWorldOpinions(buildWorldOpinions(data.wo));
+	const { countries, countryById, narratives, sources } = wo;
+
 	const modes: { id: MapMode; label: string }[] = [
 		{ id: 'liberal', label: 'what "liberal" means' },
 		{ id: 'divide', label: 'what left vs right is about' },