stats step 3: elections, party positions and opinion feeds (schema, collectors, timers); world-opinions data moved into stats.wo_*
All checks were successful
Deploy production / deploy (push) Successful in 1m12s
All checks were successful
Deploy production / deploy (push) Successful in 1m12s
This commit is contained in:
@@ -17,6 +17,13 @@ statistics into the `stats` schema with `scripts/stats-fetch.ts`:
|
||||
| `ustreasury` | `stats-ustreasury.timer` | daily 23:25 | US Treasury MTS receipts/outlays + Debt to the Penny (last 400 days; `--backfill` all) |
|
||||
| `usaspending` | `stats-usaspending.timer` | Sundays 06:10 | USAspending budgetary resources per agency and fiscal year |
|
||||
| `eponline` | `stats-eponline.timer` | 3rd of the month 05:20 | EP-Online monthly total file (RVO energy labels): homes per label for NL, Gouda and Gouda PC4s; aggregates only. Needs `EP_ONLINE_API_KEY` in `~/.config/stats/ep-online.env` (mode 600), which the runner exports for this source only |
|
||||
| `elections` | `stats-elections.timer` | 15th of the month 04:50 | A14: Kiesraad Tweede Kamer results 2010→ (data.overheid.nl CSV + EML zips read with HTTP ranges; national, municipality, Gouda polling stations), CBS 37278 (1918–2006), MIT US presidential by state. Finished elections are skipped unless `--force` |
|
||||
| `parties` | `stats-parties.timer` | quarterly, 16th 04:40 | A15: CHES trend file + V-Party (RData via `~/.local/share/stats-py`, pyreadr) linked with Party Facts into `wo_party`/`wo_party_position` |
|
||||
| `wikipolls` | `stats-wikipolls.timer` | daily 08:10 | A16: Wikipedia poll tables (NL seats, Trump approval, Lee Jae-myung approval, KR party polls); wikitext snapshot per revision |
|
||||
| `gtrends` | `stats-gtrends.timer` | daily 21:45 | A16: Google Trends trending-now RSS for NL, US, KR |
|
||||
| `gdelt` | `stats-gdelt.timer` | nightly 02:30 | A16: GDELT DOC timelinevol, 10 countries × 12 themes, last 30 days, ≥5.5 s between requests |
|
||||
| `eurobarometer` | `stats-eurobarometer.timer` | 20th of the month 05:35 | A16: Standard Eurobarometer volume A xlsx (QA3/QA4/QA5 most important issues, trust in institutions); new waves only |
|
||||
| `whr` | `stats-whr.timer` | 21st of the month 05:10 | A16: World Happiness Report Figure 2.1 (life evaluation and components, 2011→) |
|
||||
| `cbscw` | `stats-cbscw.timer` | Tuesdays 05:50 | CBS Curaçao: scrapes the topic pages (cbs.cw), matches each table's file by regex, re-imports only when its sha256 changes (`stats.cw_cbs_file`); CPI from PDF (pdfjs), GDP, government finance, population since 1900, LFS, tourism from xlsx |
|
||||
| `cwintl` | `stats-cwintl.timer` | Wednesdays 05:50 | Curaçao from international APIs: ILOSTAT SDMX (labour, with retries), OWID grapher CSVs (population, life expectancy, median age, CO₂), UN SDG area 531, UN Comtrade public preview reporter 531. EU trade with Curaçao (Eurostat `ext_lt_maineu`) runs with the `eurostat` timer |
|
||||
| – | – | manual | CBCS (centralbank.cw) blocks scripts with a Cloudflare challenge: download the BoP xlsx by hand and load it into `stats.cw_cbcs_bop` (period, item, mln_ang) |
|
||||
|
||||
11
deploy/stats-collector/stats-elections.timer
Normal file
11
deploy/stats-collector/stats-elections.timer
Normal file
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=/stats collector: Kiesraad TK results, CBS 37278 history, MIT US presidential (new elections only)
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-15 04:50:00 Europe/Amsterdam
|
||||
RandomizedDelaySec=20min
|
||||
Persistent=true
|
||||
Unit=stats-collector@elections.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
11
deploy/stats-collector/stats-eurobarometer.timer
Normal file
11
deploy/stats-collector/stats-eurobarometer.timer
Normal file
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=/stats collector: Standard Eurobarometer volume A, new waves (monthly check)
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-20 05:35:00 Europe/Amsterdam
|
||||
RandomizedDelaySec=20min
|
||||
Persistent=true
|
||||
Unit=stats-collector@eurobarometer.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
11
deploy/stats-collector/stats-gdelt.timer
Normal file
11
deploy/stats-collector/stats-gdelt.timer
Normal file
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=/stats collector: GDELT theme coverage, slow loop at 1 request per 5.5 s (nightly)
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 02:30:00 Europe/Amsterdam
|
||||
RandomizedDelaySec=20min
|
||||
Persistent=true
|
||||
Unit=stats-collector@gdelt.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
11
deploy/stats-collector/stats-gtrends.timer
Normal file
11
deploy/stats-collector/stats-gtrends.timer
Normal file
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=/stats collector: Google Trends trending-now RSS for NL, US, KR (daily)
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 21:45:00 Europe/Amsterdam
|
||||
RandomizedDelaySec=20min
|
||||
Persistent=true
|
||||
Unit=stats-collector@gtrends.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
11
deploy/stats-collector/stats-parties.timer
Normal file
11
deploy/stats-collector/stats-parties.timer
Normal file
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=/stats collector: CHES + V-Party party positions via Party Facts (quarterly)
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-01,04,07,10-16 04:40:00 Europe/Amsterdam
|
||||
RandomizedDelaySec=20min
|
||||
Persistent=true
|
||||
Unit=stats-collector@parties.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
11
deploy/stats-collector/stats-whr.timer
Normal file
11
deploy/stats-collector/stats-whr.timer
Normal file
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=/stats collector: World Happiness Report Figure 2.1 (monthly check)
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-21 05:10:00 Europe/Amsterdam
|
||||
RandomizedDelaySec=20min
|
||||
Persistent=true
|
||||
Unit=stats-collector@whr.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
11
deploy/stats-collector/stats-wikipolls.timer
Normal file
11
deploy/stats-collector/stats-wikipolls.timer
Normal file
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=/stats collector: Wikipedia opinion-poll tables (daily)
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 08:10:00 Europe/Amsterdam
|
||||
RandomizedDelaySec=20min
|
||||
Persistent=true
|
||||
Unit=stats-collector@wikipolls.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
193
drizzle/20261025220251_stats_step3/migration.sql
Normal file
193
drizzle/20261025220251_stats_step3/migration.sql
Normal file
@@ -0,0 +1,193 @@
|
||||
CREATE TABLE "stats"."eb_item" (
|
||||
"wave" text,
|
||||
"fieldwork_end" date,
|
||||
"topic" text,
|
||||
"qcode" text NOT NULL,
|
||||
"item" text,
|
||||
"country" text,
|
||||
"share" double precision,
|
||||
CONSTRAINT "eb_item_pkey" PRIMARY KEY("wave","topic","item","country")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."elec_election" (
|
||||
"election_id" text PRIMARY KEY,
|
||||
"country" char(3) NOT NULL,
|
||||
"kind" text NOT NULL,
|
||||
"date" date NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"source_id" text NOT NULL,
|
||||
"total_seats" integer
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."elec_result" (
|
||||
"election_id" text,
|
||||
"level" text,
|
||||
"area_code" text,
|
||||
"area_name" text,
|
||||
"party" text,
|
||||
"votes" bigint,
|
||||
"seats" integer,
|
||||
CONSTRAINT "elec_result_pkey" PRIMARY KEY("election_id","level","area_code","party")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."elec_turnout" (
|
||||
"election_id" text,
|
||||
"level" text,
|
||||
"area_code" text,
|
||||
"area_name" text,
|
||||
"electorate" bigint,
|
||||
"ballots" bigint,
|
||||
"valid" bigint,
|
||||
"blank" bigint,
|
||||
"invalid" bigint,
|
||||
CONSTRAINT "elec_turnout_pkey" PRIMARY KEY("election_id","level","area_code")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."gdelt_theme_vol" (
|
||||
"date" date,
|
||||
"country" char(3),
|
||||
"theme" text,
|
||||
"vol_pct" double precision,
|
||||
CONSTRAINT "gdelt_theme_vol_pkey" PRIMARY KEY("date","country","theme")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."gt_trending" (
|
||||
"geo" text,
|
||||
"date" date,
|
||||
"rank" integer NOT NULL,
|
||||
"query" text,
|
||||
"approx_traffic" integer,
|
||||
"pub_date" timestamp with time zone,
|
||||
"news" text,
|
||||
CONSTRAINT "gt_trending_pkey" PRIMARY KEY("geo","date","query")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."op_poll" (
|
||||
"series" text,
|
||||
"poll_key" text,
|
||||
"fieldwork_end" date NOT NULL,
|
||||
"pollster" text NOT NULL,
|
||||
"sample" integer,
|
||||
"option" text,
|
||||
"value" double precision,
|
||||
"seen_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "op_poll_pkey" PRIMARY KEY("series","poll_key","option")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."op_wiki_snapshot" (
|
||||
"page" text,
|
||||
"revid" bigint,
|
||||
"fetched_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"wikitext" text NOT NULL,
|
||||
CONSTRAINT "op_wiki_snapshot_pkey" PRIMARY KEY("page","revid")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."whr_score" (
|
||||
"year" integer,
|
||||
"country" text,
|
||||
"iso3" char(3),
|
||||
"rank" integer,
|
||||
"ladder" double precision,
|
||||
"lower" double precision,
|
||||
"upper" double precision,
|
||||
"gdp" double precision,
|
||||
"social" double precision,
|
||||
"health" double precision,
|
||||
"freedom" double precision,
|
||||
"generosity" double precision,
|
||||
"corruption" double precision,
|
||||
"dystopia" double precision,
|
||||
CONSTRAINT "whr_score_pkey" PRIMARY KEY("year","country")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."wo_compass_override" (
|
||||
"party_id" text PRIMARY KEY,
|
||||
"x" double precision NOT NULL,
|
||||
"y" double precision NOT NULL,
|
||||
"note" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."wo_country" (
|
||||
"iso3" char(3) PRIMARY KEY,
|
||||
"id" text NOT NULL UNIQUE,
|
||||
"iso_num" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"region" text NOT NULL,
|
||||
"liberal_meaning" text,
|
||||
"main_divide" text,
|
||||
"note" text,
|
||||
"profile" jsonb,
|
||||
"sort" integer NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."wo_cultural_map" (
|
||||
"iso3" char(3),
|
||||
"wave" text,
|
||||
"year_from" integer,
|
||||
"year_to" integer,
|
||||
"trad_secular" double precision,
|
||||
"surv_selfexp" double precision,
|
||||
"n" integer,
|
||||
"source" text,
|
||||
CONSTRAINT "wo_cultural_map_pkey" PRIMARY KEY("iso3","wave")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."wo_narrative" (
|
||||
"sort" integer PRIMARY KEY,
|
||||
"event" text NOT NULL,
|
||||
"years" text NOT NULL,
|
||||
"views" jsonb NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."wo_party" (
|
||||
"party_id" text PRIMARY KEY,
|
||||
"iso3" char(3),
|
||||
"name" text,
|
||||
"abbrev" text,
|
||||
"family" text,
|
||||
"ches_id" integer,
|
||||
"manifesto_id" integer,
|
||||
"vparty_id" integer,
|
||||
"partyfacts_id" integer,
|
||||
"sort" integer
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."wo_party_position" (
|
||||
"party_id" text,
|
||||
"year" integer,
|
||||
"source" text,
|
||||
"dim" text,
|
||||
"value" double precision,
|
||||
"vote_share" double precision,
|
||||
CONSTRAINT "wo_party_position_pkey" PRIMARY KEY("party_id","year","source","dim")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."wo_source" (
|
||||
"sort" integer PRIMARY KEY,
|
||||
"name" text NOT NULL,
|
||||
"href" text NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "stats"."wo_value_score" (
|
||||
"iso3" char(3),
|
||||
"wave" text,
|
||||
"item_code" text,
|
||||
"item_label" text,
|
||||
"stat" text,
|
||||
"value" double precision,
|
||||
"n" integer,
|
||||
CONSTRAINT "wo_value_score_pkey" PRIMARY KEY("iso3","wave","item_code","stat")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX "elec_result_area_idx" ON "stats"."elec_result" ("level","area_code");--> statement-breakpoint
|
||||
CREATE INDEX "op_poll_series_date_idx" ON "stats"."op_poll" ("series","fieldwork_end");--> statement-breakpoint
|
||||
CREATE INDEX "wo_party_iso3_idx" ON "stats"."wo_party" ("iso3");--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA stats TO personal_app;
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA stats TO personal_ro;
|
||||
END IF;
|
||||
END $$;
|
||||
35669
drizzle/20261025220251_stats_step3/snapshot.json
Normal file
35669
drizzle/20261025220251_stats_step3/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
239
drizzle/20261025220252_stats_wo_seed/migration.sql
Normal file
239
drizzle/20261025220252_stats_wo_seed/migration.sql
Normal file
@@ -0,0 +1,239 @@
|
||||
-- Seed of /projects/world-opinions, generated from src/lib/world-opinions/data.ts (commit 030f3c0).
|
||||
-- Curated text and compass placements copied exactly; the DB is the source of truth from here on.
|
||||
INSERT INTO "stats"."wo_country" ("iso3","id","iso_num","name","region","liberal_meaning","main_divide","note","profile","sort") VALUES
|
||||
('SWE', 'se', '752', 'Sweden', 'nordics', 'market', 'economy', NULL, '{"center": "High-trust, high-tax welfare state; very secular and individualist", "fights": ["immigration", "gang violence", "private welfare", "NATO"], "timeline": [{"year": "1814", "label": "last war fought"}, {"year": "1932", "label": "Social Democrats'' era"}, {"year": "1939", "label": "neutral in WWII"}, {"year": "2015", "label": "record asylum intake"}, {"year": "2024", "label": "joins NATO"}], "blindSpots": ["consensus makes disagreement taboo", "neutrality myth (iron to Germany)"]}'::jsonb, 0),
|
||||
('DNK', 'dk', '208', 'Denmark', 'nordics', 'market', 'economy', '"Venstre" (literally "Left") is the liberal, center-right party', NULL, 1),
|
||||
('NOR', 'no', '578', 'Norway', 'nordics', 'market', 'economy', NULL, NULL, 2),
|
||||
('FIN', 'fi', '246', 'Finland', 'nordics', 'market', 'economy', NULL, NULL, 3),
|
||||
('NLD', 'nl', '528', 'Netherlands', 'western-europe', 'market', 'economy', 'Liberal VVD is the main center-right party', '{"center": "Coalition consensus (\"polder model\") around a strong welfare state", "fights": ["immigration", "housing", "nitrogen & farmers", "EU"], "timeline": [{"year": "1600s", "label": "Golden Age, trade empire"}, {"year": "1940", "label": "German occupation"}, {"year": "1949", "label": "loses Indonesia"}, {"year": "1960s", "label": "pillars break, secularization"}, {"year": "2002", "label": "Fortuyn, immigration debate"}], "blindSpots": ["tolerance self-image vs discrimination", "colonial past taught thinly"]}'::jsonb, 4),
|
||||
('BEL', 'be', '056', 'Belgium', 'western-europe', 'market', 'economy', 'Language (Flemish vs French-speaking) splits every party family', NULL, 5),
|
||||
('DEU', 'de', '276', 'Germany', 'western-europe', 'market', 'economy', 'Liberal FDP = pro-market, low taxes', '{"center": "Social market economy, fiscal caution, \"never again\"", "fights": ["migration", "energy & climate", "East vs West", "debt brake"], "timeline": [{"year": "1923", "label": "hyperinflation"}, {"year": "1933", "label": "Nazi dictatorship"}, {"year": "1949", "label": "split + economic miracle"}, {"year": "1990", "label": "reunification"}, {"year": "2015", "label": "refugee crisis"}], "blindSpots": ["debt as moral failing", "Russian gas as \"just trade\""]}'::jsonb, 6),
|
||||
('AUT', 'at', '040', 'Austria', 'western-europe', 'market', 'economy', NULL, NULL, 7),
|
||||
('CHE', 'ch', '756', 'Switzerland', 'western-europe', 'market', 'economy', NULL, NULL, 8),
|
||||
('GBR', 'gb', '826', 'United Kingdom', 'western-europe', 'market', 'economy', 'Liberal Democrats sit in the center; Brexit cut across old lines', '{"center": "Pragmatic and market-friendly, but the NHS is sacred", "fights": ["Brexit", "class", "North vs South", "Scotland"], "timeline": [{"year": "1900", "label": "largest empire in history"}, {"year": "1940", "label": "\"stood alone\""}, {"year": "1948", "label": "NHS founded"}, {"year": "1979", "label": "Thatcher"}, {"year": "2016", "label": "Brexit vote"}], "blindSpots": ["empire taught selectively", "overrates the \"special relationship\""]}'::jsonb, 9),
|
||||
('IRL', 'ie', '372', 'Ireland', 'western-europe', 'market', 'history', 'The two big parties descend from opposite sides of the 1922 civil war', NULL, 10),
|
||||
('FRA', 'fr', '250', 'France', 'western-europe', 'market', 'economy', '"Libéral" means pro-market, often used as a criticism', '{"center": "Strong central state, laïcité, protest as normal politics", "fights": ["pensions", "Paris vs periphery", "Islam & laïcité", "EU"], "timeline": [{"year": "1789", "label": "Revolution invents left/right"}, {"year": "1940", "label": "defeat and Vichy"}, {"year": "1962", "label": "loses Algerian war"}, {"year": "1968", "label": "May ''68"}, {"year": "2018", "label": "Yellow Vests"}], "blindSpots": ["colorblind universalism hides racism", "influence in former African colonies"]}'::jsonb, 11),
|
||||
('ESP', 'es', '724', 'Spain', 'western-europe', 'market', 'economy', 'Regional nationalism (Catalonia, Basque Country) is a second axis', NULL, 12),
|
||||
('PRT', 'pt', '620', 'Portugal', 'western-europe', 'market', 'economy', NULL, NULL, 13),
|
||||
('ITA', 'it', '380', 'Italy', 'western-europe', 'market', 'economy', NULL, NULL, 14),
|
||||
('GRC', 'gr', '300', 'Greece', 'eastern-europe', 'market', 'economy', NULL, NULL, 15),
|
||||
('USA', 'us', '840', 'United States', 'north-america', 'left', 'culture', '"Liberal" = Democrat; the Democrats sit near Europe''s center economically', '{"center": "Individual liberty and suspicion of government, on both sides", "fights": ["abortion", "guns", "race", "immigration", "religion"], "timeline": [{"year": "1776", "label": "revolt against a distant state"}, {"year": "1861", "label": "civil war over slavery"}, {"year": "1945", "label": "wins WWII, superpower"}, {"year": "1964", "label": "Civil Rights Act"}, {"year": "1991", "label": "wins the Cold War"}], "blindSpots": ["assumes its left/right is universal", "partisan media bubbles", "little view of other systems"]}'::jsonb, 16),
|
||||
('CAN', 'ca', '124', 'Canada', 'north-america', 'left', 'economy', NULL, NULL, 17),
|
||||
('MEX', 'mx', '484', 'Mexico', 'latin-america', 'market', 'economy', NULL, NULL, 18),
|
||||
('BRA', 'br', '076', 'Brazil', 'latin-america', 'market', 'economy', 'Bolsonaro''s party is literally called "Partido Liberal"', '{"center": "Inequality makes redistribution central; religious society", "fights": ["Lula vs Bolsonaro", "crime", "Amazon vs agribusiness"], "timeline": [{"year": "1888", "label": "last to abolish slavery"}, {"year": "1964", "label": "military coup"}, {"year": "1985", "label": "democracy returns"}, {"year": "1994", "label": "ends hyperinflation"}, {"year": "2014", "label": "Car Wash scandal"}], "blindSpots": ["\"racial democracy\" myth", "WhatsApp information bubbles"]}'::jsonb, 19),
|
||||
('ARG', 'ar', '032', 'Argentina', 'latin-america', 'market', 'economy', 'President Milei calls himself a "liberal-libertarian"', NULL, 20),
|
||||
('CHL', 'cl', '152', 'Chile', 'latin-america', 'market', 'economy', NULL, NULL, 21),
|
||||
('COL', 'co', '170', 'Colombia', 'latin-america', NULL, 'economy', NULL, NULL, 22),
|
||||
('PER', 'pe', '604', 'Peru', 'latin-america', 'market', 'economy', NULL, NULL, 23),
|
||||
('VEN', 've', '862', 'Venezuela', 'latin-america', NULL, 'none', NULL, NULL, 24),
|
||||
('AUS', 'au', '036', 'Australia', 'oceania', 'market', 'economy', 'The Liberal Party is the main conservative party', NULL, 25),
|
||||
('NZL', 'nz', '554', 'New Zealand', 'oceania', 'market', 'economy', NULL, NULL, 26),
|
||||
('POL', 'pl', '616', 'Poland', 'eastern-europe', 'western', 'culture', NULL, '{"center": "Pro-NATO and wary of Russia across all parties", "fights": ["Catholic vs liberal", "abortion", "rule of law", "EU"], "timeline": [{"year": "1795", "label": "erased by partition"}, {"year": "1939", "label": "invaded by Germany and USSR"}, {"year": "1980", "label": "Solidarity movement"}, {"year": "1989", "label": "communism falls"}, {"year": "2004", "label": "joins the EU"}], "blindSpots": ["victimhood crowds out dark chapters", "distrust of Germany"]}'::jsonb, 27),
|
||||
('CZE', 'cz', '203', 'Czechia', 'eastern-europe', 'western', 'culture', NULL, NULL, 28),
|
||||
('HUN', 'hu', '348', 'Hungary', 'eastern-europe', 'western', 'culture', 'Orbán calls his model "illiberal democracy"', NULL, 29),
|
||||
('ROU', 'ro', '642', 'Romania', 'eastern-europe', 'western', 'culture', NULL, NULL, 30),
|
||||
('SRB', 'rs', '688', 'Serbia', 'eastern-europe', 'western', 'history', NULL, NULL, 31),
|
||||
('UKR', 'ua', '804', 'Ukraine', 'eastern-europe', NULL, 'history', 'Politics long split between pro-Western and pro-Russian orientation', NULL, 32),
|
||||
('RUS', 'ru', '643', 'Russia', 'eastern-europe', 'western', 'none', '"Liberal" = the pro-Western opposition, mostly in exile or prison', '{"center": "Strong state and great-power status; opposition repressed", "fights": ["largely suppressed in public"], "timeline": [{"year": "1812", "label": "Napoleon invades"}, {"year": "1941", "label": "Nazi invasion, ~27M dead"}, {"year": "1991", "label": "USSR collapses"}, {"year": "1990s", "label": "chaos and poverty"}, {"year": "2022", "label": "full invasion of Ukraine"}], "blindSpots": ["state-controlled media", "neighbors seen as sphere of influence", "Soviet crimes downplayed"]}'::jsonb, 33),
|
||||
('KAZ', 'kz', '398', 'Kazakhstan', 'eastern-europe', NULL, 'none', NULL, NULL, 34),
|
||||
('TUR', 'tr', '792', 'Turkey', 'middle-east', 'western', 'culture', 'Main divide: religious-conservative vs secular', '{"center": "National pride, strong state, balancing West and region", "fights": ["religious vs secular", "Kurdish question", "inflation"], "timeline": [{"year": "1915", "label": "Armenian genocide (denied)"}, {"year": "1920", "label": "Sèvres, fear of partition"}, {"year": "1923", "label": "Atatürk''s secular republic"}, {"year": "1980", "label": "military coup"}, {"year": "2016", "label": "failed coup"}], "blindSpots": ["genocide denial", "restricted press", "Western criticism read as hostility"]}'::jsonb, 35),
|
||||
('IRN', 'ir', '364', 'Iran', 'middle-east', 'western', 'none', NULL, NULL, 36),
|
||||
('IRQ', 'iq', '368', 'Iraq', 'middle-east', NULL, 'culture', NULL, NULL, 37),
|
||||
('EGY', 'eg', '818', 'Egypt', 'middle-east', NULL, 'none', NULL, NULL, 38),
|
||||
('MAR', 'ma', '504', 'Morocco', 'middle-east', NULL, NULL, NULL, NULL, 39),
|
||||
('SAU', 'sa', '682', 'Saudi Arabia', 'middle-east', NULL, 'none', NULL, NULL, 40),
|
||||
('NGA', 'ng', '566', 'Nigeria', 'africa', NULL, NULL, NULL, NULL, 41),
|
||||
('ETH', 'et', '231', 'Ethiopia', 'africa', NULL, NULL, NULL, NULL, 42),
|
||||
('KEN', 'ke', '404', 'Kenya', 'africa', NULL, NULL, NULL, NULL, 43),
|
||||
('ZAF', 'za', '710', 'South Africa', 'africa', NULL, 'history', 'Politics still shaped by apartheid and the liberation struggle', NULL, 44),
|
||||
('IND', 'in', '356', 'India', 'south-asia', NULL, 'culture', 'Main divide: Hindu nationalism vs secular pluralism', '{"center": "Largest democracy; religion, caste and language drive politics", "fights": ["Hindu nationalism", "caste quotas", "language", "center vs states"], "timeline": [{"year": "1858", "label": "British Raj"}, {"year": "1947", "label": "independence and Partition"}, {"year": "1961", "label": "Non-Aligned Movement"}, {"year": "1991", "label": "economic liberalization"}, {"year": "2014", "label": "BJP majority"}], "blindSpots": ["polarized media", "reduced to poverty & religion by outsiders"]}'::jsonb, 45),
|
||||
('PAK', 'pk', '586', 'Pakistan', 'south-asia', NULL, NULL, NULL, NULL, 46),
|
||||
('BGD', 'bd', '050', 'Bangladesh', 'south-asia', NULL, 'history', NULL, NULL, 47),
|
||||
('CHN', 'cn', '156', 'China', 'east-asia', 'western', 'none', '"Liberal" = pro-Western intellectuals, a label the state treats with suspicion', '{"center": "Order, growth and national rejuvenation, led by the Party", "fights": ["youth jobs", "property crisis", "local corruption"], "timeline": [{"year": "1839", "label": "Opium Wars, \"humiliation\""}, {"year": "1937", "label": "Japanese invasion"}, {"year": "1966", "label": "Cultural Revolution"}, {"year": "1978", "label": "reform and opening"}, {"year": "1989", "label": "Tiananmen (unspoken)"}], "blindSpots": ["censorship shapes history", "criticism read as new humiliation"]}'::jsonb, 48),
|
||||
('JPN', 'jp', '392', 'Japan', 'east-asia', 'market', 'history', 'The "Liberal Democratic Party" is the conservative party in power since 1955', '{"center": "Stability and harmony; one party governing since 1955", "fights": ["pacifist constitution", "aging & immigration", "war memory"], "timeline": [{"year": "1868", "label": "Meiji modernization"}, {"year": "1937", "label": "war in Asia"}, {"year": "1945", "label": "atomic bombs, defeat"}, {"year": "1947", "label": "pacifist constitution"}, {"year": "1990", "label": "bubble bursts"}], "blindSpots": ["wartime atrocities taught thinly", "sees itself as war victim"]}'::jsonb, 49),
|
||||
('KOR', 'kr', '410', 'South Korea', 'east-asia', 'left', 'history', 'Divide is mostly about North Korea, Japan and the US', '{"center": "Fierce democracy won by protest; hyper-competitive society", "fights": ["North Korea", "Japan", "gender", "chaebol power"], "timeline": [{"year": "1910", "label": "Japanese colonization"}, {"year": "1950", "label": "Korean War"}, {"year": "1980", "label": "Gwangju massacre"}, {"year": "1987", "label": "democracy"}, {"year": "2024", "label": "martial law crisis"}], "blindSpots": ["US and Japan seen through party lens"]}'::jsonb, 50),
|
||||
('TWN', 'tw', '158', 'Taiwan', 'east-asia', NULL, 'history', 'Main divide: how close to be with mainland China', NULL, 51),
|
||||
('VNM', 'vn', '704', 'Vietnam', 'southeast-asia', NULL, 'none', NULL, NULL, 52),
|
||||
('THA', 'th', '764', 'Thailand', 'southeast-asia', NULL, 'history', 'Main divide: monarchy and military vs reformists', NULL, 53),
|
||||
('MYS', 'my', '458', 'Malaysia', 'southeast-asia', NULL, 'culture', NULL, NULL, 54),
|
||||
('PHL', 'ph', '608', 'Philippines', 'southeast-asia', NULL, NULL, NULL, NULL, 55),
|
||||
('IDN', 'id', '360', 'Indonesia', 'southeast-asia', NULL, 'culture', NULL, NULL, 56);
|
||||
--> statement-breakpoint
|
||||
INSERT INTO "stats"."wo_cultural_map" ("iso3","wave","trad_secular","surv_selfexp","source") VALUES
|
||||
('SWE', 'curated', 1.8, 2.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('DNK', 'curated', 1.2, 2.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('NOR', 'curated', 1.4, 2.1, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('FIN', 'curated', 1, 1.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('NLD', 'curated', 1.1, 2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('BEL', 'curated', 0.8, 1.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('DEU', 'curated', 1.4, 1.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('AUT', 'curated', 0.6, 1.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('CHE', 'curated', 0.7, 1.9, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('GBR', 'curated', 0.3, 1.8, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('IRL', 'curated', -0.4, 1.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('FRA', 'curated', 0.8, 1.1, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('ESP', 'curated', 0.3, 1.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('PRT', 'curated', -0.3, 0.8, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('ITA', 'curated', 0, 0.5, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('GRC', 'curated', 0, 0.2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('USA', 'curated', -0.5, 1.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('CAN', 'curated', 0, 1.8, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('MEX', 'curated', -1.3, 0.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('BRA', 'curated', -1, 0.1, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('ARG', 'curated', -0.6, 0.7, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('CHL', 'curated', -0.3, 0.8, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('COL', 'curated', -1.6, 0.5, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('PER', 'curated', -1.1, 0.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('VEN', 'curated', -1.5, 0.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('AUS', 'curated', 0.2, 2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('NZL', 'curated', 0.1, 2.2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('POL', 'curated', -0.6, 0.2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('CZE', 'curated', 1.2, 0.5, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('HUN', 'curated', 0.4, -0.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('ROU', 'curated', -0.6, -0.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('SRB', 'curated', 0.1, -0.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('UKR', 'curated', 0.2, -0.9, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('RUS', 'curated', 0.6, -1, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('KAZ', 'curated', 0.4, -0.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('TUR', 'curated', -0.9, -0.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('IRN', 'curated', -0.6, -0.9, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('IRQ', 'curated', -1.3, -1.2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('EGY', 'curated', -1.5, -1.5, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('MAR', 'curated', -1.5, -1.2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('NGA', 'curated', -1.8, -0.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('ETH', 'curated', -1.5, -0.9, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('KEN', 'curated', -1.5, -0.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('ZAF', 'curated', -1.2, 0, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('IND', 'curated', -0.5, -0.5, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('PAK', 'curated', -1.4, -1, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('BGD', 'curated', -1.1, -1, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('CHN', 'curated', 1.5, -0.9, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('JPN', 'curated', 1.8, 0.4, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('KOR', 'curated', 1.2, -0.3, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('TWN', 'curated', 1.3, 0, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('VNM', 'curated', 0.5, -0.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('THA', 'curated', -0.4, 0, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('MYS', 'curated', -0.9, -0.6, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('PHL', 'curated', -1.4, -0.2, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map'),
|
||||
('IDN', 'curated', -1.2, -0.8, 'Hand-placed in the World Opinions guide after the WVS/EVS cultural map');
|
||||
--> statement-breakpoint
|
||||
INSERT INTO "stats"."wo_narrative" ("sort","event","years","views") VALUES
|
||||
(0, 'World War II', '1939–45', '[{"country": "us", "story": "D-Day: democracy beats fascism"}, {"country": "ru", "story": "We broke Hitler, 27M dead"}, {"country": "cn", "story": "Our war began in 1937"}, {"country": "pl", "story": "Liberation became Soviet occupation"}, {"country": "jp", "story": "Hiroshima: we suffered"}]'::jsonb),
|
||||
(1, 'End of the Cold War', '1989–91', '[{"country": "us", "story": "We won: the end of history"}, {"country": "pl", "story": "We freed ourselves"}, {"country": "ru", "story": "Collapse and humiliation"}, {"country": "cn", "story": "A warning: never lose control"}]'::jsonb),
|
||||
(2, 'Colonialism', '1500s–1970s', '[{"country": "nl", "story": "Golden Age trade, now debated"}, {"country": "gb", "story": "Empire, remembered fondly by many"}, {"country": "in", "story": "Extraction and Partition"}, {"country": "cn", "story": "Century of humiliation"}, {"country": "br", "story": "Conquest, then US dominance"}]'::jsonb),
|
||||
(3, 'Iraq War', '2003', '[{"country": "us", "story": "Disarmament, later a mistake"}, {"country": "fr", "story": "We warned you"}, {"country": "iq", "story": "Invasion, then civil war"}, {"country": "ru", "story": "Proof the rules are selective"}]'::jsonb);
|
||||
--> statement-breakpoint
|
||||
INSERT INTO "stats"."wo_source" ("sort","name","href") VALUES
|
||||
(0, 'World Values Survey / EVS cultural map', 'https://www.worldvaluessurvey.org/WVSContents.jsp?CMSID=Findings'),
|
||||
(1, 'Chapel Hill Expert Survey', 'https://www.chesdata.eu/'),
|
||||
(2, 'Manifesto Project', 'https://manifesto-project.wzb.eu/'),
|
||||
(3, 'Pew Research Center', 'https://www.pewresearch.org/global/'),
|
||||
(4, 'Natural Earth (map shapes)', 'https://www.naturalearthdata.com/');
|
||||
--> statement-breakpoint
|
||||
INSERT INTO "stats"."wo_party" ("party_id","iso3","name","abbrev","sort") VALUES
|
||||
('wo:us:Dem', 'USA', 'Democratic Party', 'Dem', 0),
|
||||
('wo:us:GOP', 'USA', 'Republican Party', 'GOP', 1),
|
||||
('wo:nl:GL-PvdA', 'NLD', 'GroenLinks-PvdA', 'GL-PvdA', 2),
|
||||
('wo:nl:SP', 'NLD', 'Socialistische Partij', 'SP', 3),
|
||||
('wo:nl:D66', 'NLD', 'Democraten 66', 'D66', 4),
|
||||
('wo:nl:CDA', 'NLD', 'Christen-Democratisch Appèl', 'CDA', 5),
|
||||
('wo:nl:VVD', 'NLD', 'Volkspartij voor Vrijheid en Democratie', 'VVD', 6),
|
||||
('wo:nl:BBB', 'NLD', 'BoerBurgerBeweging', 'BBB', 7),
|
||||
('wo:nl:PVV', 'NLD', 'Partij voor de Vrijheid', 'PVV', 8),
|
||||
('wo:de:Linke', 'DEU', 'Die Linke', 'Linke', 9),
|
||||
('wo:de:Grüne', 'DEU', 'Bündnis 90/Die Grünen', 'Grüne', 10),
|
||||
('wo:de:SPD', 'DEU', 'Sozialdemokratische Partei', 'SPD', 11),
|
||||
('wo:de:BSW', 'DEU', 'Bündnis Sahra Wagenknecht', 'BSW', 12),
|
||||
('wo:de:FDP', 'DEU', 'Freie Demokratische Partei', 'FDP', 13),
|
||||
('wo:de:CDU', 'DEU', 'CDU/CSU', 'CDU', 14),
|
||||
('wo:de:AfD', 'DEU', 'Alternative für Deutschland', 'AfD', 15),
|
||||
('wo:gb:Green', 'GBR', 'Green Party', 'Green', 16),
|
||||
('wo:gb:Lab', 'GBR', 'Labour', 'Lab', 17),
|
||||
('wo:gb:LD', 'GBR', 'Liberal Democrats', 'LD', 18),
|
||||
('wo:gb:Con', 'GBR', 'Conservatives', 'Con', 19),
|
||||
('wo:gb:Reform', 'GBR', 'Reform UK', 'Reform', 20),
|
||||
('wo:fr:LFI', 'FRA', 'La France Insoumise', 'LFI', 21),
|
||||
('wo:fr:PS', 'FRA', 'Parti Socialiste', 'PS', 22),
|
||||
('wo:fr:RE', 'FRA', 'Renaissance', 'RE', 23),
|
||||
('wo:fr:LR', 'FRA', 'Les Républicains', 'LR', 24),
|
||||
('wo:fr:RN', 'FRA', 'Rassemblement National', 'RN', 25),
|
||||
('wo:se:V', 'SWE', 'Vänsterpartiet', 'V', 26),
|
||||
('wo:se:S', 'SWE', 'Socialdemokraterna', 'S', 27),
|
||||
('wo:se:C', 'SWE', 'Centerpartiet', 'C', 28),
|
||||
('wo:se:M', 'SWE', 'Moderaterna', 'M', 29),
|
||||
('wo:se:SD', 'SWE', 'Sverigedemokraterna', 'SD', 30),
|
||||
('wo:pl:Lewica', 'POL', 'Lewica', 'Lewica', 31),
|
||||
('wo:pl:KO', 'POL', 'Koalicja Obywatelska', 'KO', 32),
|
||||
('wo:pl:PiS', 'POL', 'Prawo i Sprawiedliwość', 'PiS', 33),
|
||||
('wo:pl:Konf', 'POL', 'Konfederacja', 'Konf', 34),
|
||||
('wo:jp:JCP', 'JPN', 'Japanese Communist Party', 'JCP', 35),
|
||||
('wo:jp:CDP', 'JPN', 'Constitutional Democratic Party', 'CDP', 36),
|
||||
('wo:jp:Komeito', 'JPN', 'Komeito', 'Komeito', 37),
|
||||
('wo:jp:LDP', 'JPN', 'Liberal Democratic Party', 'LDP', 38),
|
||||
('wo:jp:Ishin', 'JPN', 'Nippon Ishin no Kai', 'Ishin', 39),
|
||||
('wo:jp:Sansei', 'JPN', 'Sanseitō', 'Sansei', 40),
|
||||
('wo:kr:Justice', 'KOR', 'Justice Party', 'Justice', 41),
|
||||
('wo:kr:DPK', 'KOR', 'Democratic Party of Korea', 'DPK', 42),
|
||||
('wo:kr:PPP', 'KOR', 'People Power Party', 'PPP', 43),
|
||||
('wo:in:INC', 'IND', 'Indian National Congress', 'INC', 44),
|
||||
('wo:in:BJP', 'IND', 'Bharatiya Janata Party', 'BJP', 45),
|
||||
('wo:br:PT', 'BRA', 'Partido dos Trabalhadores', 'PT', 46),
|
||||
('wo:br:MDB', 'BRA', 'Movimento Democrático Brasileiro', 'MDB', 47),
|
||||
('wo:br:PL', 'BRA', 'Partido Liberal', 'PL', 48),
|
||||
('wo:ca:NDP', 'CAN', 'New Democratic Party', 'NDP', 49),
|
||||
('wo:ca:LPC', 'CAN', 'Liberal Party of Canada', 'LPC', 50),
|
||||
('wo:ca:CPC', 'CAN', 'Conservative Party of Canada', 'CPC', 51);
|
||||
--> statement-breakpoint
|
||||
INSERT INTO "stats"."wo_compass_override" ("party_id","x","y","note") VALUES
|
||||
('wo:us:Dem', 4.8, 3.5, 'The US "liberals": near Europe''s center economically'),
|
||||
('wo:us:GOP', 7.8, 8, NULL),
|
||||
('wo:nl:GL-PvdA', 2.5, 1.5, NULL),
|
||||
('wo:nl:SP', 1.5, 5, NULL),
|
||||
('wo:nl:D66', 5.5, 1.5, NULL),
|
||||
('wo:nl:CDA', 6, 6, NULL),
|
||||
('wo:nl:VVD', 7.5, 5, 'Dutch "liberals": market-liberal, center-right'),
|
||||
('wo:nl:BBB', 6, 7.5, NULL),
|
||||
('wo:nl:PVV', 4.5, 9, 'Radical right on immigration, centrist on welfare'),
|
||||
('wo:de:Linke', 1, 2.5, NULL),
|
||||
('wo:de:Grüne', 3, 1.5, NULL),
|
||||
('wo:de:SPD', 3.5, 3.5, NULL),
|
||||
('wo:de:BSW', 2, 7, NULL),
|
||||
('wo:de:FDP', 8.5, 3, 'German "liberals": socially open, strongly pro-market'),
|
||||
('wo:de:CDU', 6.5, 6, NULL),
|
||||
('wo:de:AfD', 7, 9.5, NULL),
|
||||
('wo:gb:Green', 1.5, 1.5, NULL),
|
||||
('wo:gb:Lab', 3.8, 3.5, NULL),
|
||||
('wo:gb:LD', 5, 2.5, NULL),
|
||||
('wo:gb:Con', 7, 6.5, NULL),
|
||||
('wo:gb:Reform', 7.5, 8.5, NULL),
|
||||
('wo:fr:LFI', 1, 2.5, NULL),
|
||||
('wo:fr:PS', 3, 3, NULL),
|
||||
('wo:fr:RE', 6.5, 4, NULL),
|
||||
('wo:fr:LR', 7.5, 7, NULL),
|
||||
('wo:fr:RN', 4.5, 9, 'Economically left of Macron: lower retirement age, protectionism'),
|
||||
('wo:se:V', 1, 2, NULL),
|
||||
('wo:se:S', 3, 3.5, NULL),
|
||||
('wo:se:C', 7, 2.5, NULL),
|
||||
('wo:se:M', 7.5, 5.5, NULL),
|
||||
('wo:se:SD', 5.5, 9, NULL),
|
||||
('wo:pl:Lewica', 2, 2, NULL),
|
||||
('wo:pl:KO', 6.5, 4, NULL),
|
||||
('wo:pl:PiS', 3.5, 9, 'Culturally right, economically left (big family benefits)'),
|
||||
('wo:pl:Konf', 9, 9, NULL),
|
||||
('wo:jp:JCP', 1, 3, NULL),
|
||||
('wo:jp:CDP', 4, 3.5, NULL),
|
||||
('wo:jp:Komeito', 5, 5.5, NULL),
|
||||
('wo:jp:LDP', 6, 7, 'Called "Liberal", governs as conservatives since 1955'),
|
||||
('wo:jp:Ishin', 7.5, 6, NULL),
|
||||
('wo:jp:Sansei', 6, 9, NULL),
|
||||
('wo:kr:Justice', 1.5, 1.5, NULL),
|
||||
('wo:kr:DPK', 4, 4.5, 'The "progressive" camp, cautious on social issues'),
|
||||
('wo:kr:PPP', 7, 7, NULL),
|
||||
('wo:in:INC', 4.5, 4, NULL),
|
||||
('wo:in:BJP', 6, 8.5, 'Hindu-nationalist, pro-business, big welfare schemes'),
|
||||
('wo:br:PT', 2.5, 4.5, NULL),
|
||||
('wo:br:MDB', 6, 6, NULL),
|
||||
('wo:br:PL', 7.5, 9, NULL),
|
||||
('wo:ca:NDP', 2.5, 2.5, NULL),
|
||||
('wo:ca:LPC', 4.8, 3, NULL),
|
||||
('wo:ca:CPC', 7.5, 7, NULL);
|
||||
35669
drizzle/20261025220252_stats_wo_seed/snapshot.json
Normal file
35669
drizzle/20261025220252_stats_wo_seed/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -177,7 +177,7 @@ try {
|
||||
throw new Error(`groceries schema grants are wrong: ${JSON.stringify(groceries)}`);
|
||||
// Public /stats section: open data, so the app role (and the host-side stats collector, which
|
||||
// uses it) reads and writes, and personal_ro may read. Bump the count with every new stats table.
|
||||
const STATS_TABLES = 32;
|
||||
const STATS_TABLES = 49;
|
||||
const [stats] = await client`
|
||||
select
|
||||
count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
// bun scripts/stats-fetch.ts eponline [--force] (needs EP_ONLINE_API_KEY)
|
||||
// bun scripts/stats-fetch.ts cbscw [--force] [--only=cpi,gdp,…] CBS Curaçao files
|
||||
// bun scripts/stats-fetch.ts cwintl [--only=ilostat,owid,unsdg,comtrade] Curaçao via international APIs
|
||||
// bun scripts/stats-fetch.ts elections [--force] [--only=kiesraad,cbs,mit] A14
|
||||
// bun scripts/stats-fetch.ts parties CHES + V-Party + Party Facts (A15)
|
||||
// bun scripts/stats-fetch.ts wikipolls|eurobarometer [--force]|whr|gtrends|gdelt A16
|
||||
// bun scripts/stats-fetch.ts normalise-cbs rebuild CBS chart series from the raw rows
|
||||
import { ensureMeta, sql } from './stats/common';
|
||||
import { cbs, normaliseCbs } from './stats/cbs';
|
||||
@@ -28,6 +31,13 @@ import { usaspending } from './stats/usaspending';
|
||||
import { eponline } from './stats/eponline';
|
||||
import { cwintl } from './stats/cwintl';
|
||||
import { cwcbs } from './stats/cwcbs';
|
||||
import { elections } from './stats/elections';
|
||||
import { parties } from './stats/parties';
|
||||
import { wikipolls } from './stats/wikipolls';
|
||||
import { eurobarometer } from './stats/eurobarometer';
|
||||
import { whr } from './stats/whr';
|
||||
import { gtrends } from './stats/gtrends';
|
||||
import { gdelt } from './stats/gdelt';
|
||||
|
||||
const [source, ...args] = process.argv.slice(2);
|
||||
const flag = (name: string) => args.includes(`--${name}`);
|
||||
@@ -82,9 +92,30 @@ try {
|
||||
case 'eponline':
|
||||
await eponline({ force: flag('force') || flag('backfill'), job });
|
||||
break;
|
||||
case 'elections':
|
||||
await elections({ force: flag('force') || flag('backfill'), only: value('only'), job });
|
||||
break;
|
||||
case 'parties':
|
||||
await parties({ job });
|
||||
break;
|
||||
case 'wikipolls':
|
||||
await wikipolls({ job });
|
||||
break;
|
||||
case 'eurobarometer':
|
||||
await eurobarometer({ force: flag('force') || flag('backfill'), job });
|
||||
break;
|
||||
case 'whr':
|
||||
await whr({ job });
|
||||
break;
|
||||
case 'gtrends':
|
||||
await gtrends({ job });
|
||||
break;
|
||||
case 'gdelt':
|
||||
await gdelt({ job, timespan: value('timespan') });
|
||||
break;
|
||||
default:
|
||||
throw new Error(
|
||||
`Unknown source "${source}" (cbs, eurostat, fred, bis, worldbank, imf, owid, energycharts, rijksfinancien, ustreasury, usaspending, eponline, cbscw, cwintl, normalise-cbs)`
|
||||
`Unknown source "${source}" (cbs, eurostat, fred, bis, worldbank, imf, owid, energycharts, rijksfinancien, ustreasury, usaspending, eponline, cbscw, cwintl, elections, parties, wikipolls, eurobarometer, whr, gtrends, gdelt, normalise-cbs)`
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
|
||||
@@ -37,7 +37,17 @@ const hostGapMs: Record<string, number> = {
|
||||
'cuatro.sim-cdn.nl': 1000,
|
||||
'sdmx.ilo.org': 3000,
|
||||
'unstats.un.org': 1500,
|
||||
'comtradeapi.un.org': 2500
|
||||
'comtradeapi.un.org': 2500,
|
||||
'data.overheid.nl': 500,
|
||||
'dataverse.harvard.edu': 1000,
|
||||
'github.com': 1000,
|
||||
'partyfacts.herokuapp.com': 2000,
|
||||
'en.wikipedia.org': 1000,
|
||||
'data.europa.eu': 1000,
|
||||
'webgate.ec.europa.eu': 2000,
|
||||
'files.worldhappiness.report': 1000,
|
||||
'trends.google.com': 3000,
|
||||
'api.gdeltproject.org': 5500
|
||||
};
|
||||
const lastHit = new Map<string, number>();
|
||||
|
||||
@@ -54,7 +64,7 @@ export async function http(url: string, init: RequestInit = {}, tries = 4): Prom
|
||||
response = await fetch(url, {
|
||||
...init,
|
||||
headers: { 'user-agent': USER_AGENT, accept: '*/*', ...(init.headers ?? {}) },
|
||||
signal: AbortSignal.timeout(180_000)
|
||||
signal: init.signal ?? AbortSignal.timeout(180_000)
|
||||
});
|
||||
if (response.ok) return response;
|
||||
if (response.status !== 429 && response.status < 500) {
|
||||
@@ -248,6 +258,74 @@ export const SOURCES = [
|
||||
licence: 'Free use with attribution (UN Comtrade public preview)',
|
||||
key_required: false,
|
||||
attribution: 'Source: UN Comtrade Database (United Nations Statistics Division)'
|
||||
},
|
||||
{
|
||||
source_id: 'kiesraad',
|
||||
name: 'Kiesraad election results (data.overheid.nl)',
|
||||
base_url: 'https://data.overheid.nl/dataset?q=verkiezingsuitslag',
|
||||
licence: 'CC0 1.0',
|
||||
key_required: false,
|
||||
attribution: 'Source: Kiesraad, verkiezingsuitslagen (data.overheid.nl), CC0'
|
||||
},
|
||||
{
|
||||
source_id: 'mitelection',
|
||||
name: 'MIT Election Data and Science Lab, U.S. President 1976–2024',
|
||||
base_url: 'https://doi.org/10.7910/DVN/42MVDX',
|
||||
licence: 'CC0 1.0',
|
||||
key_required: false,
|
||||
attribution:
|
||||
'Source: MIT Election Data and Science Lab, U.S. President 1976–2024 (Harvard Dataverse), CC0'
|
||||
},
|
||||
{
|
||||
source_id: 'ches',
|
||||
name: 'Chapel Hill Expert Survey (trend file) + V-Party + Party Facts',
|
||||
base_url: 'https://www.chesdata.eu',
|
||||
licence:
|
||||
'Free for non-commercial research use with citation (CHES); V-Dem CC BY-SA 4.0; Party Facts CC BY-SA 4.0',
|
||||
key_required: false,
|
||||
attribution:
|
||||
'Sources: Chapel Hill Expert Survey 1999–2024 (Jolly et al.); V-Dem V-Party v2 (Lührmann et al.); Party Facts (Döring & Regel)'
|
||||
},
|
||||
{
|
||||
source_id: 'wikipedia',
|
||||
name: 'Wikipedia opinion-polling tables',
|
||||
base_url: 'https://en.wikipedia.org',
|
||||
licence: 'CC BY-SA 4.0',
|
||||
key_required: false,
|
||||
attribution:
|
||||
'Source: Wikipedia opinion-polling articles and their cited pollsters, CC BY-SA 4.0'
|
||||
},
|
||||
{
|
||||
source_id: 'eurobarometer',
|
||||
name: 'Standard Eurobarometer (volume A tables)',
|
||||
base_url: 'https://data.europa.eu',
|
||||
licence: 'Reuse authorised with acknowledgement (Commission Decision 2011/833/EU)',
|
||||
key_required: false,
|
||||
attribution: 'Source: European Commission, Standard Eurobarometer, © European Union'
|
||||
},
|
||||
{
|
||||
source_id: 'whr',
|
||||
name: 'World Happiness Report (Figure 2.1 data)',
|
||||
base_url: 'https://worldhappiness.report',
|
||||
licence: 'Free reuse with attribution',
|
||||
key_required: false,
|
||||
attribution: 'Source: World Happiness Report (Wellbeing Research Centre, Gallup World Poll)'
|
||||
},
|
||||
{
|
||||
source_id: 'gtrends',
|
||||
name: 'Google Trends, trending now (RSS)',
|
||||
base_url: 'https://trends.google.com/trending/rss',
|
||||
licence: 'Google Terms of Service; shown with attribution',
|
||||
key_required: false,
|
||||
attribution: 'Source: Google Trends'
|
||||
},
|
||||
{
|
||||
source_id: 'gdelt',
|
||||
name: 'GDELT Project DOC 2.0 API',
|
||||
base_url: 'https://api.gdeltproject.org/api/v2/doc/doc',
|
||||
licence: 'Open, unrestricted use with citation',
|
||||
key_required: false,
|
||||
attribution: 'Source: The GDELT Project'
|
||||
}
|
||||
] as const;
|
||||
|
||||
|
||||
543
scripts/stats/elections.ts
Normal file
543
scripts/stats/elections.ts
Normal file
@@ -0,0 +1,543 @@
|
||||
// A14 elections: NL Tweede Kamer from the Kiesraad open data (data.overheid.nl, CC0) and CBS
|
||||
// 37278 (1918–2006), US presidential results by state from the MIT Election Lab (CC0).
|
||||
// - Kiesraad: per election the "CSV formaat" zip (votes per list per municipality and per
|
||||
// polling station; 2023+ also turnout and seats) plus the EML zips, read with HTTP ranges, for
|
||||
// the official result (seats, election date) and, before 2023, the national total count.
|
||||
// Polling stations are kept for Gouda (GM0513) only.
|
||||
// - Results are final, so an election already loaded is skipped unless --force.
|
||||
import { insertChunks, getJson, http, log, run, sql } from './common';
|
||||
import { parseCsv } from './owid';
|
||||
import { RemoteZip } from './remotezip';
|
||||
|
||||
const CKAN =
|
||||
'https://data.overheid.nl/data/api/3/action/package_search?q=verkiezingsuitslag%20Tweede%20Kamer&rows=100';
|
||||
const STATION_MUNIS = new Set(['GM0513']);
|
||||
|
||||
type Result = {
|
||||
election_id: string;
|
||||
level: string;
|
||||
area_code: string;
|
||||
area_name: string | null;
|
||||
party: string;
|
||||
votes: number | null;
|
||||
seats: number | null;
|
||||
};
|
||||
type Turnout = {
|
||||
election_id: string;
|
||||
level: string;
|
||||
area_code: string;
|
||||
area_name: string | null;
|
||||
electorate: number | null;
|
||||
ballots: number | null;
|
||||
valid: number | null;
|
||||
blank: number | null;
|
||||
invalid: number | null;
|
||||
};
|
||||
type Election = {
|
||||
election_id: string;
|
||||
country: string;
|
||||
kind: string;
|
||||
date: string;
|
||||
name: string;
|
||||
source_id: string;
|
||||
total_seats: number | null;
|
||||
};
|
||||
|
||||
async function save(e: Election, results: Result[], turnout: Turnout[]) {
|
||||
// Merge duplicate keys (a list name can repeat within a municipality in old files).
|
||||
const merged = new Map<string, Result>();
|
||||
for (const r of results) {
|
||||
const key = `${r.level}|${r.area_code}|${r.party}`;
|
||||
const prev = merged.get(key);
|
||||
if (!prev) merged.set(key, { ...r });
|
||||
else {
|
||||
prev.votes =
|
||||
prev.votes == null && r.votes == null ? null : (prev.votes ?? 0) + (r.votes ?? 0);
|
||||
prev.seats = prev.seats ?? r.seats;
|
||||
}
|
||||
}
|
||||
await sql.begin(async (tx) => {
|
||||
await tx`delete from stats.elec_result where election_id = ${e.election_id}`;
|
||||
await tx`delete from stats.elec_turnout where election_id = ${e.election_id}`;
|
||||
await tx`delete from stats.elec_election where election_id = ${e.election_id}`;
|
||||
await insertChunks(tx, 'elec_election', [e]);
|
||||
await insertChunks(tx, 'elec_result', [...merged.values()]);
|
||||
await insertChunks(tx, 'elec_turnout', turnout);
|
||||
});
|
||||
return merged.size + turnout.length;
|
||||
}
|
||||
|
||||
const num = (s: string | undefined) => {
|
||||
if (s == null || s.trim() === '') return null;
|
||||
const n = Number(s.replace(/\s/g, ''));
|
||||
return Number.isFinite(n) ? n : null;
|
||||
};
|
||||
const gm = (code: string) => `GM${code.replace(/^\D+/, '').padStart(4, '0')}`;
|
||||
const xmlText = (s: string) =>
|
||||
s
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&');
|
||||
|
||||
/** Rows of a semicolon CSV as objects keyed by the header. */
|
||||
function rowsOf(text: string): Record<string, string>[] {
|
||||
const [header, ...rows] = parseCsv(text, ';');
|
||||
return rows
|
||||
.filter((r) => r.length >= header.length - 1 && r.some((x) => x !== ''))
|
||||
.map((r) => Object.fromEntries(header.map((h, i) => [h.trim(), (r[i] ?? '').trim()])));
|
||||
}
|
||||
|
||||
/** EML 520 (Resultaat): election date and seats per list (elected candidates per affiliation). */
|
||||
function parseResultaat(xml: string) {
|
||||
const date = xml.match(/<kr:ElectionDate>([\d-]+)<\/kr:ElectionDate>/)?.[1] ?? null;
|
||||
const seats = new Map<string, number>();
|
||||
let current = '';
|
||||
for (const m of xml.matchAll(/<Selection>([\s\S]*?)<\/Selection>/g)) {
|
||||
const aff = m[1].match(/<RegisteredName>([\s\S]*?)<\/RegisteredName>/);
|
||||
if (aff) {
|
||||
current = xmlText(aff[1].trim());
|
||||
if (!seats.has(current)) seats.set(current, 0);
|
||||
} else if (/<Candidate>/.test(m[1]) && /<Elected>yes<\/Elected>/.test(m[1]))
|
||||
seats.set(current, (seats.get(current) ?? 0) + 1);
|
||||
}
|
||||
return { date, seats };
|
||||
}
|
||||
|
||||
/** EML 510d (Totaaltelling): national votes per list and the count totals. */
|
||||
function parseTotaaltelling(xml: string) {
|
||||
const total = xml.match(/<TotalVotes>([\s\S]*?)<\/TotalVotes>/)?.[1] ?? '';
|
||||
const votes = new Map<string, number>();
|
||||
for (const m of total.matchAll(
|
||||
/<Selection><AffiliationIdentifier[^>]*><RegisteredName>([\s\S]*?)<\/RegisteredName><\/AffiliationIdentifier><ValidVotes>(\d+)<\/ValidVotes>/g
|
||||
))
|
||||
votes.set(xmlText(m[1].trim()), Number(m[2]));
|
||||
const pick = (re: RegExp) => num(total.match(re)?.[1]);
|
||||
return {
|
||||
votes,
|
||||
electorate: pick(/<Cast>(\d+)<\/Cast>/),
|
||||
ballots: pick(/<TotalCounted>(\d+)<\/TotalCounted>/),
|
||||
invalid: pick(/<RejectedVotes ReasonCode="ongeldig">(\d+)</),
|
||||
blank: pick(/<RejectedVotes ReasonCode="blanco">(\d+)</)
|
||||
};
|
||||
}
|
||||
|
||||
type CkanResource = { url: string; name?: string; format?: string };
|
||||
type CkanPackage = { name: string; title: string; resources: CkanResource[] };
|
||||
|
||||
async function kiesraad(force: boolean, note: (w: string) => void) {
|
||||
const res = await getJson<{ result: { results: CkanPackage[] } }>(CKAN);
|
||||
const packages = res.result.results
|
||||
.map((p) => ({ p, year: p.name.match(/^verkiezingsuitslag-tweede-kamer-(\d{4})$/)?.[1] }))
|
||||
.filter((x): x is { p: CkanPackage; year: string } => !!x.year)
|
||||
.sort((a, b) => a.year.localeCompare(b.year));
|
||||
const loaded = new Set(
|
||||
(await sql`select election_id from stats.elec_election where source_id = 'kiesraad'`).map(
|
||||
(r) => r.election_id as string
|
||||
)
|
||||
);
|
||||
let rows = 0;
|
||||
for (const { p, year } of packages) {
|
||||
const id = `NL-TK-${year}`;
|
||||
if (loaded.has(id) && !force) continue;
|
||||
const csvRes = p.resources.find((r) => /csv/i.test(decodeURIComponent(r.url)));
|
||||
if (!csvRes) {
|
||||
note(`${id}: no CSV zip`);
|
||||
continue;
|
||||
}
|
||||
const csvZip = await RemoteZip.open(csvRes.url);
|
||||
// EML zips: find the official result (and, for older years, the national count).
|
||||
let resultaat: ReturnType<typeof parseResultaat> | null = null;
|
||||
let totaal: ReturnType<typeof parseTotaaltelling> | null = null;
|
||||
for (const r of p.resources.filter((r) => r !== csvRes && /\.zip$/i.test(r.url))) {
|
||||
const zip = await RemoteZip.open(r.url);
|
||||
const re = zip.find(/(^|\/)Resultaat_TK\d+[^/]*\.eml\.xml$/i)[0];
|
||||
if (re && !resultaat) resultaat = parseResultaat(await zip.text(re));
|
||||
const tt = zip.find(/(^|\/)Totaaltelling_TK\d+[^/]*\.eml\.xml$/i)[0];
|
||||
if (tt && !totaal && !csvZip.find(/_uitslag\.csv$/i).length)
|
||||
totaal = parseTotaaltelling(await zip.text(tt));
|
||||
if (resultaat && (totaal || csvZip.find(/_uitslag\.csv$/i).length)) break;
|
||||
}
|
||||
const results: Result[] = [];
|
||||
const turnout: Turnout[] = [];
|
||||
const base = { election_id: id };
|
||||
const uitslag = csvZip.find(/_uitslag\.csv$/i)[0];
|
||||
if (uitslag) {
|
||||
for (const r of rowsOf(await csvZip.text(uitslag))) {
|
||||
const code = r.RegioCode ?? '';
|
||||
const level = code.startsWith('L') ? 'national' : /^[GO]/.test(code) ? 'muni' : null;
|
||||
if (!level) continue;
|
||||
const area_code = level === 'national' ? 'NL' : gm(code);
|
||||
const area_name = level === 'national' ? 'Nederland' : r.Regio;
|
||||
const v = num(r.Waarde);
|
||||
if (r.VeldType === 'LijstAantalStemmen' && r.LijstNaam)
|
||||
results.push({
|
||||
...base,
|
||||
level,
|
||||
area_code,
|
||||
area_name,
|
||||
party: r.LijstNaam,
|
||||
votes: v,
|
||||
seats: null
|
||||
});
|
||||
else if (r.VeldType === 'LijstAantalZetels' && r.LijstNaam && level === 'national')
|
||||
results.push({
|
||||
...base,
|
||||
level,
|
||||
area_code,
|
||||
area_name,
|
||||
party: r.LijstNaam,
|
||||
votes: null,
|
||||
seats: v
|
||||
});
|
||||
else if (
|
||||
[
|
||||
'Kiesgerechtigden',
|
||||
'Opkomst',
|
||||
'AantalGeldigeStemmen',
|
||||
'AantalBlancoStemmen',
|
||||
'AantalOngeldigeStemmen'
|
||||
].includes(r.VeldType)
|
||||
) {
|
||||
let t = turnout.find((x) => x.level === level && x.area_code === area_code);
|
||||
if (!t) {
|
||||
t = {
|
||||
...base,
|
||||
level,
|
||||
area_code,
|
||||
area_name,
|
||||
electorate: null,
|
||||
ballots: null,
|
||||
valid: null,
|
||||
blank: null,
|
||||
invalid: null
|
||||
};
|
||||
turnout.push(t);
|
||||
}
|
||||
const col = {
|
||||
Kiesgerechtigden: 'electorate',
|
||||
Opkomst: 'ballots',
|
||||
AantalGeldigeStemmen: 'valid',
|
||||
AantalBlancoStemmen: 'blank',
|
||||
AantalOngeldigeStemmen: 'invalid'
|
||||
}[r.VeldType] as 'electorate' | 'ballots' | 'valid' | 'blank' | 'invalid';
|
||||
t[col] = v;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const gem = csvZip.find(/Per_Gemeente\.csv$/i)[0];
|
||||
if (!gem) throw new Error(`${id}: no municipality CSV in ${csvRes.url}`);
|
||||
for (const r of rowsOf(await csvZip.text(gem)))
|
||||
results.push({
|
||||
...base,
|
||||
level: 'muni',
|
||||
area_code: gm(r.GemeenteCode),
|
||||
area_name: r.GemeenteNaam,
|
||||
party: r.PartijNaam,
|
||||
votes: num(r.AantalStemmen),
|
||||
seats: null
|
||||
});
|
||||
if (totaal) {
|
||||
const seats = resultaat?.seats ?? new Map<string, number>();
|
||||
for (const [party, votes] of totaal.votes)
|
||||
results.push({
|
||||
...base,
|
||||
level: 'national',
|
||||
area_code: 'NL',
|
||||
area_name: 'Nederland',
|
||||
party,
|
||||
votes,
|
||||
seats: seats.get(party) ?? null
|
||||
});
|
||||
const valid = [...totaal.votes.values()].reduce((a, b) => a + b, 0);
|
||||
turnout.push({
|
||||
...base,
|
||||
level: 'national',
|
||||
area_code: 'NL',
|
||||
area_name: 'Nederland',
|
||||
electorate: totaal.electorate,
|
||||
ballots: totaal.ballots,
|
||||
valid,
|
||||
blank: totaal.blank,
|
||||
invalid: totaal.invalid
|
||||
});
|
||||
} else note(`${id}: no Totaaltelling EML, national total missing`);
|
||||
}
|
||||
// Merge votes and seats rows of the national level (2023+ lists them separately).
|
||||
const nat = new Map<string, Result>();
|
||||
for (const r of results.filter((r) => r.level === 'national')) {
|
||||
const prev = nat.get(r.party);
|
||||
if (prev) {
|
||||
prev.votes ??= r.votes;
|
||||
prev.seats ??= r.seats;
|
||||
} else nat.set(r.party, r);
|
||||
}
|
||||
const finalResults = [...results.filter((r) => r.level !== 'national'), ...nat.values()];
|
||||
// Polling stations (Gouda only).
|
||||
const sb = csvZip.find(/Per_Stembureau\.csv$/i)[0];
|
||||
if (sb) {
|
||||
for (const r of rowsOf(await csvZip.text(sb))) {
|
||||
const muni = gm(r.GemeenteCode);
|
||||
if (!STATION_MUNIS.has(muni)) continue;
|
||||
finalResults.push({
|
||||
...base,
|
||||
level: 'station',
|
||||
area_code: `${muni}-${r.StembureauCode}`,
|
||||
area_name: r.StembureauNaam || null,
|
||||
party: r.PartijNaam,
|
||||
votes: num(r.AantalStemmen),
|
||||
seats: null
|
||||
});
|
||||
}
|
||||
} else note(`${id}: no polling-station CSV`);
|
||||
const totalSeats = [...nat.values()].reduce((a, r) => a + (r.seats ?? 0), 0) || null;
|
||||
const date = resultaat?.date ?? KNOWN_DATES[year];
|
||||
if (!date) throw new Error(`${id}: no election date`);
|
||||
if (!resultaat) note(`${id}: no Resultaat EML (seats/date from CSV or table)`);
|
||||
const n = await save(
|
||||
{
|
||||
election_id: id,
|
||||
country: 'NLD',
|
||||
kind: 'TK',
|
||||
date,
|
||||
name: `Tweede Kamer ${year}`,
|
||||
source_id: 'kiesraad',
|
||||
total_seats: totalSeats
|
||||
},
|
||||
finalResults,
|
||||
turnout
|
||||
);
|
||||
log(
|
||||
`kiesraad ${id}: ${n} rows (${finalResults.filter((r) => r.level === 'station').length} Gouda station rows)`
|
||||
);
|
||||
rows += n;
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// Election days (CBS 37278 only has the year). Source: Kiesraad / Parlement.com.
|
||||
const KNOWN_DATES: Record<string, string> = {
|
||||
'1918': '1918-07-03',
|
||||
'1922': '1922-07-05',
|
||||
'1925': '1925-07-01',
|
||||
'1929': '1929-07-03',
|
||||
'1933': '1933-04-26',
|
||||
'1937': '1937-05-26',
|
||||
'1946': '1946-05-17',
|
||||
'1948': '1948-07-07',
|
||||
'1952': '1952-06-25',
|
||||
'1956': '1956-06-13',
|
||||
'1959': '1959-03-12',
|
||||
'1963': '1963-05-15',
|
||||
'1967': '1967-02-15',
|
||||
'1971': '1971-04-28',
|
||||
'1972': '1972-11-29',
|
||||
'1977': '1977-05-25',
|
||||
'1981': '1981-05-26',
|
||||
'1982': '1982-09-08',
|
||||
'1986': '1986-05-21',
|
||||
'1989': '1989-09-06',
|
||||
'1994': '1994-05-03',
|
||||
'1998': '1998-05-06',
|
||||
'2002': '2002-05-15',
|
||||
'2003': '2003-01-22',
|
||||
'2006': '2006-11-22',
|
||||
'2010': '2010-06-09',
|
||||
'2012': '2012-09-12',
|
||||
'2017': '2017-03-15',
|
||||
'2021': '2021-03-17',
|
||||
'2023': '2023-11-22',
|
||||
'2025': '2025-10-29'
|
||||
};
|
||||
|
||||
type CbsProp = { Key: string; Title: string; Type: string; ParentID?: number | null; ID: number };
|
||||
|
||||
async function cbs37278() {
|
||||
const base = 'https://opendata.cbs.nl/ODataApi/odata/37278';
|
||||
const props = (await getJson<{ value: CbsProp[] }>(`${base}/DataProperties`)).value;
|
||||
const groups = new Map(props.filter((p) => p.Type === 'TopicGroup').map((p) => [p.ID, p.Title]));
|
||||
const data = (
|
||||
await getJson<{ value: Record<string, string | number | null>[] }>(`${base}/TypedDataSet`)
|
||||
).value;
|
||||
let rows = 0;
|
||||
for (const d of data) {
|
||||
const year = String(d.Perioden).slice(0, 4);
|
||||
const id = `NL-TK-${year}`;
|
||||
const results = new Map<string, Result>();
|
||||
const turnout: Turnout = {
|
||||
election_id: id,
|
||||
level: 'national',
|
||||
area_code: 'NL',
|
||||
area_name: 'Nederland',
|
||||
electorate: null,
|
||||
ballots: null,
|
||||
valid: null,
|
||||
blank: null,
|
||||
invalid: null
|
||||
};
|
||||
for (const p of props.filter((p) => p.Type === 'Topic')) {
|
||||
const v = d[p.Key] == null ? null : Number(d[p.Key]);
|
||||
const group = groups.get(p.ParentID ?? -1) ?? '';
|
||||
if (p.Key.startsWith('Kiesgerechtigden')) turnout.electorate = v;
|
||||
else if (p.Key.startsWith('TotaalStemmen')) turnout.ballots = v;
|
||||
else if (p.Key.startsWith('OngeldigeStemmen')) turnout.invalid = v;
|
||||
else if (p.Key.startsWith('TotaalGeldigeStemmen')) turnout.valid = v;
|
||||
else if (/^(Opkomst|TotaalAantalZetels)/.test(p.Key)) continue;
|
||||
else if (v != null && /stemmen per partij/i.test(group)) {
|
||||
const r = results.get(p.Title) ?? {
|
||||
election_id: id,
|
||||
level: 'national',
|
||||
area_code: 'NL',
|
||||
area_name: 'Nederland',
|
||||
party: p.Title,
|
||||
votes: null,
|
||||
seats: null
|
||||
};
|
||||
r.votes = v;
|
||||
results.set(p.Title, r);
|
||||
} else if (v != null && /zetel/i.test(group)) {
|
||||
const r = results.get(p.Title) ?? {
|
||||
election_id: id,
|
||||
level: 'national',
|
||||
area_code: 'NL',
|
||||
area_name: 'Nederland',
|
||||
party: p.Title,
|
||||
votes: null,
|
||||
seats: null
|
||||
};
|
||||
r.seats = v;
|
||||
results.set(p.Title, r);
|
||||
}
|
||||
}
|
||||
const list = [...results.values()].filter((r) => r.votes || r.seats);
|
||||
const totalSeats = list.reduce((a, r) => a + (r.seats ?? 0), 0) || null;
|
||||
rows += await save(
|
||||
{
|
||||
election_id: id,
|
||||
country: 'NLD',
|
||||
kind: 'TK',
|
||||
date: KNOWN_DATES[year] ?? `${year}-01-01`,
|
||||
name: `Tweede Kamer ${year}`,
|
||||
source_id: 'cbs',
|
||||
total_seats: totalSeats
|
||||
},
|
||||
list,
|
||||
[turnout]
|
||||
);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** US presidential election day: the Tuesday after the first Monday in November. */
|
||||
function usElectionDay(year: number) {
|
||||
const d = new Date(Date.UTC(year, 10, 1));
|
||||
const firstMonday = 1 + ((8 - d.getUTCDay()) % 7);
|
||||
return `${year}-11-${String(firstMonday + 1).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
async function mit(note: (w: string) => void) {
|
||||
const ds = await getJson<{
|
||||
data: { latestVersion: { files: { dataFile: { id: number; filename: string } }[] } };
|
||||
}>(
|
||||
'https://dataverse.harvard.edu/api/datasets/:persistentId/?persistentId=doi:10.7910/DVN/42MVDX'
|
||||
);
|
||||
const file = ds.data.latestVersion.files
|
||||
.map((f) => f.dataFile)
|
||||
.filter((f) => /^\d{4}-\d{4}-president\.(csv|tab)$/.test(f.filename))
|
||||
.sort((a, b) => b.filename.localeCompare(a.filename))[0];
|
||||
if (!file) throw new Error('MIT president file not found in doi:10.7910/DVN/42MVDX');
|
||||
const text = await (
|
||||
await http(`https://dataverse.harvard.edu/api/access/datafile/${file.id}`)
|
||||
).text();
|
||||
const sep = file.filename.endsWith('.tab') ? '\t' : ',';
|
||||
const [header, ...lines] = parseCsv(text, sep);
|
||||
const col = (name: string) => header.indexOf(name);
|
||||
const [cYear, cState, cPo, cParty, cVotes, cTotal] = [
|
||||
'year',
|
||||
'state',
|
||||
'state_po',
|
||||
'party_simplified',
|
||||
'candidatevotes',
|
||||
'totalvotes'
|
||||
].map(col);
|
||||
if ([cYear, cState, cPo, cParty, cVotes, cTotal].some((i) => i < 0))
|
||||
throw new Error(`MIT columns changed: ${header.join(',')}`);
|
||||
const byYear = new Map<number, { results: Map<string, Result>; turnout: Map<string, Turnout> }>();
|
||||
for (const l of lines) {
|
||||
if (l.length < header.length - 1) continue;
|
||||
const year = Number(l[cYear]);
|
||||
const id = `US-PRES-${year}`;
|
||||
const y = byYear.get(year) ?? { results: new Map(), turnout: new Map() };
|
||||
byYear.set(year, y);
|
||||
const state = l[cPo];
|
||||
const party = l[cParty] || 'OTHER';
|
||||
const votes = Number(l[cVotes]) || 0;
|
||||
for (const [level, area_code, area_name] of [
|
||||
['state', state, l[cState]],
|
||||
['national', 'US', 'United States']
|
||||
]) {
|
||||
const key = `${level}|${area_code}|${party}`;
|
||||
const r = y.results.get(key) ?? {
|
||||
election_id: id,
|
||||
level,
|
||||
area_code,
|
||||
area_name,
|
||||
party,
|
||||
votes: 0,
|
||||
seats: null
|
||||
};
|
||||
r.votes = (r.votes ?? 0) + votes;
|
||||
y.results.set(key, r);
|
||||
}
|
||||
if (!y.turnout.has(state))
|
||||
y.turnout.set(state, {
|
||||
election_id: id,
|
||||
level: 'state',
|
||||
area_code: state,
|
||||
area_name: l[cState],
|
||||
electorate: null,
|
||||
ballots: Number(l[cTotal]) || null,
|
||||
valid: null,
|
||||
blank: null,
|
||||
invalid: null
|
||||
});
|
||||
}
|
||||
let rows = 0;
|
||||
for (const [year, y] of byYear) {
|
||||
const states = [...y.turnout.values()];
|
||||
states.push({
|
||||
election_id: `US-PRES-${year}`,
|
||||
level: 'national',
|
||||
area_code: 'US',
|
||||
area_name: 'United States',
|
||||
electorate: null,
|
||||
ballots: states.reduce((a, s) => a + (s.ballots ?? 0), 0),
|
||||
valid: null,
|
||||
blank: null,
|
||||
invalid: null
|
||||
});
|
||||
rows += await save(
|
||||
{
|
||||
election_id: `US-PRES-${year}`,
|
||||
country: 'USA',
|
||||
kind: 'PRES',
|
||||
date: usElectionDay(year),
|
||||
name: `US presidential ${year}`,
|
||||
source_id: 'mitelection',
|
||||
total_seats: null
|
||||
},
|
||||
[...y.results.values()],
|
||||
states
|
||||
);
|
||||
}
|
||||
if (!byYear.size) note('MIT file had no rows');
|
||||
return rows;
|
||||
}
|
||||
|
||||
export async function elections(opts: { force: boolean; only?: string; job: string }) {
|
||||
const want = (k: string) => !opts.only || opts.only.split(',').includes(k);
|
||||
if (want('kiesraad')) await run('kiesraad', opts.job, (note) => kiesraad(opts.force, note));
|
||||
if (want('cbs')) await run('cbs', `${opts.job}:37278`, () => cbs37278());
|
||||
if (want('mit')) await run('mitelection', opts.job, (note) => mit(note));
|
||||
}
|
||||
135
scripts/stats/eurobarometer.ts
Normal file
135
scripts/stats/eurobarometer.ts
Normal file
@@ -0,0 +1,135 @@
|
||||
// A16 Standard Eurobarometer, volume A (EU + candidate countries, weighted shares), via the
|
||||
// data.europa.eu catalogue (© European Union, reuse under the Commission's reuse policy).
|
||||
// Reads QA3 (most important issues facing our country), QA4 (… facing you personally), QA5
|
||||
// (… facing the EU) and the trust batteries (QA6_x national/international institutions, QA8_x
|
||||
// EU institutions; share "tend to trust"). Only waves published as xlsx; waves already loaded are
|
||||
// skipped unless --force.
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { getJson, http, insertChunks, log, run, sql } from './common';
|
||||
import { readXlsx, type Cell } from './xlsx';
|
||||
|
||||
const SEARCH =
|
||||
'https://data.europa.eu/api/hub/search/search?q=standard%20eurobarometer&limit=100&filter=dataset';
|
||||
|
||||
type Dist = {
|
||||
title?: string | Record<string, string>;
|
||||
download_url?: string[];
|
||||
access_url?: string[];
|
||||
};
|
||||
type Dataset = { title?: string | Record<string, string>; distributions?: Dist[] };
|
||||
const en = (t: Dataset['title']) =>
|
||||
typeof t === 'string' ? t : (t?.en ?? Object.values(t ?? {})[0] ?? '');
|
||||
|
||||
type Row = {
|
||||
wave: string;
|
||||
fieldwork_end: string | null;
|
||||
topic: string;
|
||||
qcode: string;
|
||||
item: string;
|
||||
country: string;
|
||||
share: number;
|
||||
};
|
||||
|
||||
const TOPICS: [RegExp, string][] = [
|
||||
[/^QA3$/, 'mii_country'],
|
||||
[/^QA4$/, 'mii_personal'],
|
||||
[/^QA5$/, 'mii_eu'],
|
||||
[/^QA6_\d+$/, 'trust'],
|
||||
[/^QA8_\d+$/, 'trust_eu']
|
||||
];
|
||||
|
||||
const str = (c: Cell | undefined) => (c == null ? '' : String(c)).trim();
|
||||
const share = (c: Cell | undefined) => {
|
||||
const n = typeof c === 'number' ? c : Number(str(c));
|
||||
return str(c) !== '' && Number.isFinite(n) ? n : null;
|
||||
};
|
||||
|
||||
export function parseVolumeA(sheets: Awaited<ReturnType<typeof readXlsx>>): Row[] {
|
||||
const content = sheets.find((s) => s.name === 'Content');
|
||||
if (!content) throw new Error('no Content sheet');
|
||||
const wave = str(content.rows.find((r) => /^wave/i.test(str(r[0])))?.[1]);
|
||||
const fw = str(content.rows.find((r) => /^fieldwork/i.test(str(r[0])))?.[1]);
|
||||
const d = fw.match(/(\d{1,2})\/(\d{1,2})\/(\d{4})\s*$/);
|
||||
const fieldwork_end = d ? `${d[3]}-${d[2].padStart(2, '0')}-${d[1].padStart(2, '0')}` : null;
|
||||
if (!wave) throw new Error('no wave in Content sheet');
|
||||
const questions = new Map<string, string>();
|
||||
for (const r of content.rows) {
|
||||
const sheet = str(r[0])
|
||||
.match(/^"?([A-Z][A-Z0-9_]*)"?\)?$|"([A-Z][A-Z0-9_]*)"\)\s*$/)
|
||||
?.slice(1)
|
||||
.find(Boolean);
|
||||
if (sheet) questions.set(sheet, str(r[2]));
|
||||
}
|
||||
const out: Row[] = [];
|
||||
for (const s of sheets) {
|
||||
const topic = TOPICS.find(([re]) => re.test(s.name))?.[1];
|
||||
if (!topic) continue;
|
||||
const h = s.rows.findIndex(
|
||||
(r) => r.some((c) => str(c) === 'BE') && r.some((c) => str(c) === 'NL')
|
||||
);
|
||||
if (h < 0) continue;
|
||||
const countries = s.rows[h].map((c) => str(c).split('\n').at(-1) ?? '');
|
||||
// Answers come in pairs: a French row with counts, then an English row with shares.
|
||||
for (let i = h + 2; i < s.rows.length; i++) {
|
||||
const label = str(s.rows[i][1]);
|
||||
const next = s.rows[i + 1];
|
||||
if (!label || !next || !str(next[1])) continue;
|
||||
const values = countries.map((_, c) => share(next[c]));
|
||||
if (!values.some((v) => v != null && v <= 1)) continue;
|
||||
const item = str(next[1]);
|
||||
i++;
|
||||
let name = item;
|
||||
if (topic.startsWith('trust')) {
|
||||
if (!/^tend to trust/i.test(item)) continue;
|
||||
name = (questions.get(s.name) ?? s.name).split(':-').at(-1)?.trim() ?? s.name;
|
||||
}
|
||||
countries.forEach((country, c) => {
|
||||
const v = values[c];
|
||||
if (c >= 2 && country && v != null && v <= 1)
|
||||
out.push({ wave, fieldwork_end, topic, qcode: s.name, item: name, country, share: v });
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function eurobarometer(opts: { force: boolean; job: string }) {
|
||||
await run('eurobarometer', opts.job, async (note) => {
|
||||
const res = await getJson<{ result: { results: Dataset[] } }>(SEARCH);
|
||||
const waves = res.result.results
|
||||
.filter((d) => /^standard eurobarometer/i.test(en(d.title)))
|
||||
.flatMap((d) =>
|
||||
(d.distributions ?? [])
|
||||
.filter((x) => /volume[ _]A\.xlsx$/i.test(en(x.title)))
|
||||
.map((x) => ({ dataset: en(d.title), url: x.download_url?.[0] ?? x.access_url?.[0] }))
|
||||
)
|
||||
.filter((w): w is { dataset: string; url: string } => !!w.url);
|
||||
const loaded = new Set(
|
||||
(await sql`select distinct wave from stats.eb_item`).map((r) => r.wave as string)
|
||||
);
|
||||
let rows = 0;
|
||||
for (const w of waves) {
|
||||
const num = w.dataset.match(/(?:STD|Eurobarometer )(\d{2,3})/i)?.[1];
|
||||
if (num && !opts.force && [...loaded].some((l) => l.split('.')[0] === num)) continue;
|
||||
const file = join(tmpdir(), `eb-${process.pid}.xlsx`);
|
||||
await Bun.write(file, await (await http(w.url)).arrayBuffer());
|
||||
let items: Row[] = [];
|
||||
try {
|
||||
items = parseVolumeA(await readXlsx(file));
|
||||
} catch (error) {
|
||||
note(`${w.dataset}: ${error instanceof Error ? error.message : error}`);
|
||||
}
|
||||
await Bun.file(file).delete();
|
||||
if (!items.length) continue;
|
||||
const dedup = new Map(items.map((r) => [`${r.wave}|${r.topic}|${r.item}|${r.country}`, r]));
|
||||
await sql.begin(async (tx) => {
|
||||
await tx`delete from stats.eb_item where wave = ${items[0].wave}`;
|
||||
await insertChunks(tx, 'eb_item', [...dedup.values()]);
|
||||
});
|
||||
log(`eurobarometer ${items[0].wave}: ${dedup.size} rows`);
|
||||
rows += dedup.size;
|
||||
}
|
||||
return rows;
|
||||
});
|
||||
}
|
||||
86
scripts/stats/gdelt.ts
Normal file
86
scripts/stats/gdelt.ts
Normal file
@@ -0,0 +1,86 @@
|
||||
// A16 GDELT DOC 2.0 "timelinevol": daily share of monitored news coverage (by source country)
|
||||
// that mentions a GKG theme, last 30 days, upserted nightly. GDELT asks for at most one request
|
||||
// every 5 seconds, so the loop is deliberately slow (~12 minutes). GDELT data is open, cite GDELT.
|
||||
import { http, insertChunks, run, sql } from './common';
|
||||
|
||||
export const GDELT_COUNTRIES: Record<string, string> = {
|
||||
netherlands: 'NLD',
|
||||
unitedstates: 'USA',
|
||||
southkorea: 'KOR',
|
||||
germany: 'DEU',
|
||||
france: 'FRA',
|
||||
unitedkingdom: 'GBR',
|
||||
belgium: 'BEL',
|
||||
japan: 'JPN',
|
||||
india: 'IND',
|
||||
brazil: 'BRA'
|
||||
};
|
||||
export const GDELT_THEMES = [
|
||||
'ECON_INFLATION',
|
||||
'ECON_HOUSING_PRICES',
|
||||
'UNEMPLOYMENT',
|
||||
'IMMIGRATION',
|
||||
'REFUGEES',
|
||||
'ENV_CLIMATECHANGE',
|
||||
'TERROR',
|
||||
'PROTEST',
|
||||
'CORRUPTION',
|
||||
'ELECTION',
|
||||
'MILITARY',
|
||||
'HEALTH_PANDEMIC'
|
||||
];
|
||||
|
||||
export async function gdelt(opts: { job: string; timespan?: string }) {
|
||||
await run('gdelt', opts.job, async (note) => {
|
||||
let rows = 0;
|
||||
const bad = new Set<string>();
|
||||
for (const [name, iso3] of Object.entries(GDELT_COUNTRIES))
|
||||
for (const theme of GDELT_THEMES) {
|
||||
if (bad.has(theme)) continue;
|
||||
const q = encodeURIComponent(`theme:${theme} sourcecountry:${name}`);
|
||||
const url = `https://api.gdeltproject.org/api/v2/doc/doc?query=${q}&mode=timelinevol&format=json×pan=${opts.timespan ?? '30d'}`;
|
||||
let text: string;
|
||||
try {
|
||||
text = await (await http(url, {}, 3)).text();
|
||||
} catch (error) {
|
||||
note(`${theme}/${name}: ${error instanceof Error ? error.message : error}`);
|
||||
continue;
|
||||
}
|
||||
let data: { timeline?: { data: { date: string; value: number }[] }[] };
|
||||
try {
|
||||
data = JSON.parse(text);
|
||||
} catch {
|
||||
// GDELT answers invalid queries with a plain-text message.
|
||||
note(`${theme}: ${text.slice(0, 120)}`);
|
||||
if (/theme|invalid/i.test(text)) bad.add(theme);
|
||||
continue;
|
||||
}
|
||||
const points = (data.timeline?.[0]?.data ?? []).map((p) => ({
|
||||
date: `${p.date.slice(0, 4)}-${p.date.slice(4, 6)}-${p.date.slice(6, 8)}`,
|
||||
country: iso3,
|
||||
theme,
|
||||
vol_pct: p.value
|
||||
}));
|
||||
// Hourly resolution (short timespans): average per day.
|
||||
const byDay = new Map<string, { sum: number; n: number; row: (typeof points)[number] }>();
|
||||
for (const p of points) {
|
||||
const d = byDay.get(p.date) ?? { sum: 0, n: 0, row: p };
|
||||
d.sum += p.vol_pct;
|
||||
d.n++;
|
||||
byDay.set(p.date, d);
|
||||
}
|
||||
const daily = [...byDay.values()].map((d) => ({ ...d.row, vol_pct: d.sum / d.n }));
|
||||
if (!daily.length) continue;
|
||||
await sql.begin(async (tx) => {
|
||||
await insertChunks(
|
||||
tx,
|
||||
'gdelt_theme_vol',
|
||||
daily,
|
||||
'on conflict (date, country, theme) do update set vol_pct = excluded.vol_pct'
|
||||
);
|
||||
});
|
||||
rows += daily.length;
|
||||
}
|
||||
return rows;
|
||||
});
|
||||
}
|
||||
56
scripts/stats/gtrends.ts
Normal file
56
scripts/stats/gtrends.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
// A16 Google Trends "trending now" RSS (daily; terms of service: display with attribution).
|
||||
// Keeps each day's list per country; a re-run on the same day refreshes rank and traffic.
|
||||
import { getText, insertChunks, run, sql } from './common';
|
||||
import { decode } from './htmltable';
|
||||
|
||||
const GEOS = ['NL', 'US', 'KR'];
|
||||
|
||||
/** "200+", "2K+", "1M+" → lower bound as a number. */
|
||||
const traffic = (s: string | null) => {
|
||||
const m = s?.replace(/,/g, '').match(/([\d.]+)\s*([KM])?/i);
|
||||
if (!m) return null;
|
||||
return Math.round(Number(m[1]) * (m[2]?.toUpperCase() === 'M' ? 1e6 : m[2] ? 1e3 : 1));
|
||||
};
|
||||
|
||||
const tag = (xml: string, name: string) => {
|
||||
const m = xml.match(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`));
|
||||
return m ? decode(m[1].replace(/^<!\[CDATA\[|\]\]>$/g, '').trim()) : null;
|
||||
};
|
||||
|
||||
export async function gtrends(opts: { job: string }) {
|
||||
await run('gtrends', opts.job, async () => {
|
||||
let rows = 0;
|
||||
for (const geo of GEOS) {
|
||||
const xml = await getText(`https://trends.google.com/trending/rss?geo=${geo}`);
|
||||
const today = new Intl.DateTimeFormat('en-CA', { timeZone: 'Europe/Amsterdam' }).format(
|
||||
new Date()
|
||||
);
|
||||
const items = [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m, i) => {
|
||||
const news = [...m[1].matchAll(/<ht:news_item>([\s\S]*?)<\/ht:news_item>/g)]
|
||||
.map((n) => ({
|
||||
title: tag(n[1], 'ht:news_item_title'),
|
||||
source: tag(n[1], 'ht:news_item_source'),
|
||||
url: tag(n[1], 'ht:news_item_url')
|
||||
}))
|
||||
.slice(0, 3);
|
||||
const pub = tag(m[1], 'pubDate');
|
||||
return {
|
||||
geo,
|
||||
date: today,
|
||||
rank: i + 1,
|
||||
query: tag(m[1], 'title') ?? '',
|
||||
approx_traffic: traffic(tag(m[1], 'ht:approx_traffic')),
|
||||
pub_date: pub ? new Date(pub) : null,
|
||||
news: JSON.stringify(news)
|
||||
};
|
||||
});
|
||||
const unique = [...new Map(items.filter((x) => x.query).map((x) => [x.query, x])).values()];
|
||||
await sql.begin(async (tx) => {
|
||||
await tx`delete from stats.gt_trending where geo = ${geo} and date = ${today}`;
|
||||
await insertChunks(tx, 'gt_trending', unique);
|
||||
});
|
||||
rows += unique.length;
|
||||
}
|
||||
return rows;
|
||||
});
|
||||
}
|
||||
129
scripts/stats/htmltable.ts
Normal file
129
scripts/stats/htmltable.ts
Normal file
@@ -0,0 +1,129 @@
|
||||
// Minimal HTML table reader for MediaWiki output: expands rowspan/colspan into a grid of cell
|
||||
// texts (references, hidden spans and styles removed). Keeps each cell's data-sort-value.
|
||||
|
||||
export type Cell = { text: string; sort: string | null; header: boolean };
|
||||
|
||||
const ENTITIES: Record<string, string> = {
|
||||
amp: '&',
|
||||
lt: '<',
|
||||
gt: '>',
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
nbsp: ' ',
|
||||
ndash: '–',
|
||||
mdash: '—',
|
||||
minus: '−'
|
||||
};
|
||||
export const decode = (s: string) =>
|
||||
s.replace(/&(#x?[0-9a-f]+|\w+);/gi, (m, e: string) => {
|
||||
if (e[0] === '#')
|
||||
return String.fromCodePoint(
|
||||
e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1))
|
||||
);
|
||||
return ENTITIES[e.toLowerCase()] ?? m;
|
||||
});
|
||||
|
||||
export function cellText(html: string) {
|
||||
return decode(
|
||||
html
|
||||
.replace(/<sup[\s\S]*?<\/sup>/gi, '')
|
||||
.replace(/<style[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<span[^>]*display:\s*none[^>]*>[\s\S]*?<\/span>/gi, '')
|
||||
.replace(/<br\s*\/?>/gi, ' ')
|
||||
.replace(/<[^>]+>/g, '')
|
||||
)
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** All tables (outer HTML) whose opening tag matches `cls`, with the text of the nearest
|
||||
* preceding heading. Nested tables are skipped over. */
|
||||
export function findTables(html: string, cls = /wikitable/) {
|
||||
const out: { html: string; heading: string }[] = [];
|
||||
const re = /<table\b[^>]*>/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(html))) {
|
||||
if (!cls.test(m[0])) continue;
|
||||
let depth = 1;
|
||||
const tag = /<\/?table\b[^>]*>/gi;
|
||||
tag.lastIndex = m.index + m[0].length;
|
||||
let t: RegExpExecArray | null;
|
||||
while (depth && (t = tag.exec(html))) depth += t[0][1] === '/' ? -1 : 1;
|
||||
const end = tag.lastIndex;
|
||||
const headings = [...html.slice(0, m.index).matchAll(/<h[2-4][^>]*>([\s\S]*?)<\/h[2-4]>/gi)];
|
||||
out.push({ html: html.slice(m.index, end), heading: cellText(headings.at(-1)?.[1] ?? '') });
|
||||
re.lastIndex = end;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function tableGrid(table: string): Cell[][] {
|
||||
const grid: Cell[][] = [];
|
||||
const pending: (Cell & { left: number })[] = []; // by column: rowspans carried down
|
||||
const rows = table.split(/<tr\b[^>]*>/i).slice(1);
|
||||
for (const rowHtml of rows) {
|
||||
const row: Cell[] = [];
|
||||
let col = 0;
|
||||
const place = () => {
|
||||
while (pending[col] && pending[col].left > 0) {
|
||||
row[col] = pending[col];
|
||||
pending[col].left--;
|
||||
col++;
|
||||
}
|
||||
};
|
||||
for (const c of rowHtml.matchAll(/<(t[hd])\b([^>]*)>([\s\S]*?)(?=<t[hd]\b|<\/tr>|$)/gi)) {
|
||||
place();
|
||||
const attrs = c[2];
|
||||
const rowspan = Number(attrs.match(/rowspan="?(\d+)/i)?.[1] ?? 1);
|
||||
const colspan = Number(attrs.match(/colspan="?(\d+)/i)?.[1] ?? 1);
|
||||
const cell: Cell = {
|
||||
text: cellText(c[3].replace(/<\/t[hd]>[\s\S]*$/i, '')),
|
||||
sort: attrs.match(/data-sort-value="([^"]*)"/i)?.[1] ?? null,
|
||||
header: c[1].toLowerCase() === 'th'
|
||||
};
|
||||
for (let k = 0; k < colspan; k++) {
|
||||
row[col] = cell;
|
||||
if (rowspan > 1) pending[col] = { ...cell, left: rowspan - 1 };
|
||||
else delete pending[col];
|
||||
col++;
|
||||
}
|
||||
}
|
||||
place();
|
||||
if (row.length) grid.push(row);
|
||||
}
|
||||
return grid;
|
||||
}
|
||||
|
||||
const MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
|
||||
|
||||
/** End date of a fieldwork range like "13–15 Oct 2026", "Sep 28 – Oct 2, 2026", "2026-10-15". */
|
||||
export function fieldworkEnd(
|
||||
text: string,
|
||||
sort: string | null,
|
||||
fallbackYear?: number
|
||||
): string | null {
|
||||
for (const s of [sort, text]) {
|
||||
const iso = s?.match(/(\d{4})-(\d{1,2})-(\d{1,2})(?!.*\d{4}-\d{1,2}-\d{1,2})/);
|
||||
if (iso) return `${iso[1]}-${iso[2].padStart(2, '0')}-${iso[3].padStart(2, '0')}`;
|
||||
}
|
||||
const t = text.toLowerCase().replace(/[–—−]/g, '-');
|
||||
const year = Number(t.match(/(\d{4})(?!.*\d{4})/)?.[1] ?? fallbackYear);
|
||||
if (!year) return null;
|
||||
const months = [...t.matchAll(/\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?/g)];
|
||||
if (!months.length) return null;
|
||||
const last = months[months.length - 1];
|
||||
const month = MONTHS.indexOf(last[1]) + 1;
|
||||
// Day: the number right after the last month ("Oct 2") or right before it ("15 Oct").
|
||||
const after = t
|
||||
.slice(last.index! + last[0].length)
|
||||
.match(/^\s*(\d{1,2})(?:\s*-\s*(\d{1,2}))?\b(?!\d)/);
|
||||
const before = t.slice(0, last.index).match(/(\d{1,2})\s*$/);
|
||||
const day = Number(after?.[2] ?? after?.[1] ?? before?.[1] ?? 0);
|
||||
if (!day || day > 31) return null;
|
||||
return `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
export const firstNumber = (s: string) => {
|
||||
const m = s.replace(/,(?=\d{3}\b)/g, '').match(/-?\d+(?:\.\d+)?/);
|
||||
return m ? Number(m[0]) : null;
|
||||
};
|
||||
@@ -56,7 +56,7 @@ export const ENERGY_COLUMNS: Record<string, { title: string; unit: string; serie
|
||||
};
|
||||
|
||||
/** Minimal CSV parser (quoted fields, commas inside quotes). */
|
||||
export function parseCsv(text: string): string[][] {
|
||||
export function parseCsv(text: string, sep = ','): string[][] {
|
||||
const out: string[][] = [];
|
||||
let row: string[] = [];
|
||||
let field = '';
|
||||
@@ -70,7 +70,7 @@ export function parseCsv(text: string): string[][] {
|
||||
} else if (ch === '"') quoted = false;
|
||||
else field += ch;
|
||||
} else if (ch === '"') quoted = true;
|
||||
else if (ch === ',') {
|
||||
else if (ch === sep) {
|
||||
row.push(field);
|
||||
field = '';
|
||||
} else if (ch === '\n' || ch === '\r') {
|
||||
|
||||
248
scripts/stats/parties.ts
Normal file
248
scripts/stats/parties.ts
Normal file
@@ -0,0 +1,248 @@
|
||||
// A15 party positions: CHES trend file (party means 1999–2024) and V-Party v2 (V-Dem, via the
|
||||
// vdemdata GitHub repo's RData), linked through the Party Facts crosswalk. Writes stats.wo_party
|
||||
// (data-driven rows; the curated guide parties keep their own `wo:` rows) and
|
||||
// stats.wo_party_position. Curated rows get a partyfacts_id when a match is unambiguous.
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { insertChunks, http, log, run, sql } from './common';
|
||||
import { parseCsv } from './owid';
|
||||
|
||||
const CHES_URL =
|
||||
'https://github.com/chesdata/chesdata.github.io/releases/download/ches-trend/1999-2024_CHES_dataset_meansV2.csv';
|
||||
const VPARTY_URL = 'https://github.com/vdeminstitute/vdemdata/raw/master/data/vparty.RData';
|
||||
const PARTYFACTS_URL = 'https://partyfacts.herokuapp.com/download/external-parties-csv/';
|
||||
const PYTHON = process.env.STATS_PYTHON ?? `${process.env.HOME}/.local/share/stats-py/bin/python`;
|
||||
|
||||
const CHES_COUNTRY: Record<number, string> = {
|
||||
1: 'BEL',
|
||||
2: 'DNK',
|
||||
3: 'DEU',
|
||||
4: 'GRC',
|
||||
5: 'ESP',
|
||||
6: 'FRA',
|
||||
7: 'IRL',
|
||||
8: 'ITA',
|
||||
10: 'NLD',
|
||||
11: 'GBR',
|
||||
12: 'PRT',
|
||||
13: 'AUT',
|
||||
14: 'FIN',
|
||||
16: 'SWE',
|
||||
20: 'BGR',
|
||||
21: 'CZE',
|
||||
22: 'EST',
|
||||
23: 'HUN',
|
||||
24: 'LVA',
|
||||
25: 'LTU',
|
||||
26: 'POL',
|
||||
27: 'ROU',
|
||||
28: 'SVK',
|
||||
29: 'SVN',
|
||||
31: 'HRV',
|
||||
37: 'MLT',
|
||||
38: 'LUX',
|
||||
40: 'CYP',
|
||||
9: 'NOR',
|
||||
35: 'NOR',
|
||||
36: 'CHE',
|
||||
34: 'TUR',
|
||||
45: 'ISL'
|
||||
};
|
||||
const CHES_FAMILY: Record<number, string> = {
|
||||
1: 'radical right',
|
||||
2: 'conservative',
|
||||
3: 'liberal',
|
||||
4: 'christian democratic',
|
||||
5: 'socialist',
|
||||
6: 'radical left',
|
||||
7: 'green',
|
||||
8: 'regionalist',
|
||||
9: 'no family',
|
||||
10: 'confessional',
|
||||
11: 'agrarian/centre'
|
||||
};
|
||||
const CHES_DIMS = ['lrgen', 'lrecon', 'galtan', 'eu_position', 'immigrate_policy'];
|
||||
const VPARTY_DIMS = [
|
||||
'v2xpa_popul',
|
||||
'v2xpa_antiplural',
|
||||
'v2pariglef',
|
||||
'v2paimmig',
|
||||
'v2palgbt',
|
||||
'v2parelig',
|
||||
'v2pawelf',
|
||||
'v2paculsup'
|
||||
];
|
||||
|
||||
type Party = {
|
||||
party_id: string;
|
||||
iso3: string | null;
|
||||
name: string | null;
|
||||
abbrev: string | null;
|
||||
family: string | null;
|
||||
ches_id: number | null;
|
||||
manifesto_id: number | null;
|
||||
vparty_id: number | null;
|
||||
partyfacts_id: number | null;
|
||||
sort: null;
|
||||
};
|
||||
type Position = {
|
||||
party_id: string;
|
||||
year: number;
|
||||
source: string;
|
||||
dim: string;
|
||||
value: number | null;
|
||||
vote_share: number | null;
|
||||
};
|
||||
|
||||
const table = (text: string) => {
|
||||
const [header, ...rows] = parseCsv(text);
|
||||
return rows
|
||||
.filter((r) => r.length >= header.length - 1)
|
||||
.map((r) => Object.fromEntries(header.map((h, i) => [h, r[i] ?? ''])));
|
||||
};
|
||||
const num = (s: string | undefined) =>
|
||||
s == null || s === '' || s === 'NA' ? null : Number.isFinite(Number(s)) ? Number(s) : null;
|
||||
|
||||
export async function parties(opts: { job: string }) {
|
||||
await run('ches', opts.job, async (note) => {
|
||||
// Party Facts: dataset ids → partyfacts ids (and countries).
|
||||
const pf = table(await (await http(PARTYFACTS_URL)).text());
|
||||
const link = new Map<string, { pf: number | null; country: string }>();
|
||||
for (const r of pf)
|
||||
if (r.dataset_key === 'ches' || r.dataset_key === 'vparty' || r.dataset_key === 'manifesto')
|
||||
link.set(`${r.dataset_key}:${r.dataset_party_id}`, {
|
||||
pf: num(r.partyfacts_id),
|
||||
country: r.country
|
||||
});
|
||||
const partiesById = new Map<string, Party>();
|
||||
const upsert = (p: Partial<Party> & { party_id: string }) => {
|
||||
const prev = partiesById.get(p.party_id);
|
||||
if (!prev)
|
||||
partiesById.set(p.party_id, {
|
||||
iso3: null,
|
||||
name: null,
|
||||
abbrev: null,
|
||||
family: null,
|
||||
ches_id: null,
|
||||
manifesto_id: null,
|
||||
vparty_id: null,
|
||||
partyfacts_id: null,
|
||||
sort: null,
|
||||
...p
|
||||
});
|
||||
else
|
||||
for (const [k, v] of Object.entries(p))
|
||||
if (v != null && (prev as Record<string, unknown>)[k] == null)
|
||||
(prev as Record<string, unknown>)[k] = v;
|
||||
};
|
||||
const positions = new Map<string, Position>();
|
||||
const put = (p: Position) => positions.set(`${p.party_id}|${p.year}|${p.source}|${p.dim}`, p);
|
||||
|
||||
// CHES
|
||||
const ches = table(await (await http(CHES_URL)).text());
|
||||
for (const r of ches) {
|
||||
const chesId = num(r.party_id);
|
||||
if (chesId == null) continue;
|
||||
const l = link.get(`ches:${chesId}`);
|
||||
const id = l?.pf != null ? `pf:${l.pf}` : `ches:${chesId}`;
|
||||
const iso3 = CHES_COUNTRY[Number(r.country)] ?? l?.country ?? null;
|
||||
if (!iso3) note(`CHES country ${r.country} unmapped`);
|
||||
upsert({
|
||||
party_id: id,
|
||||
iso3,
|
||||
abbrev: r.party || null,
|
||||
family: CHES_FAMILY[Number(r.family)] ?? null,
|
||||
ches_id: chesId,
|
||||
manifesto_id: num(r.cmp_id),
|
||||
partyfacts_id: l?.pf ?? null
|
||||
});
|
||||
for (const dim of CHES_DIMS)
|
||||
if (r[dim] !== undefined)
|
||||
put({
|
||||
party_id: id,
|
||||
year: Number(r.year),
|
||||
source: 'ches',
|
||||
dim,
|
||||
value: num(r[dim]),
|
||||
vote_share: num(r.vote)
|
||||
});
|
||||
}
|
||||
const chesRows = positions.size;
|
||||
|
||||
// V-Party (RData → CSV via the stats Python venv)
|
||||
let vpartyRows = 0;
|
||||
try {
|
||||
const file = join(tmpdir(), `vparty-${process.pid}.RData`);
|
||||
await Bun.write(file, await (await http(VPARTY_URL)).arrayBuffer());
|
||||
const proc = Bun.spawn([PYTHON, join(import.meta.dir, 'vparty-csv.py'), file], {
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe'
|
||||
});
|
||||
const csv = await new Response(proc.stdout).text();
|
||||
const err = await new Response(proc.stderr).text();
|
||||
if ((await proc.exited) !== 0) throw new Error(`vparty-csv.py failed: ${err.slice(0, 300)}`);
|
||||
await Bun.file(file).delete();
|
||||
for (const r of table(csv)) {
|
||||
const vpId = num(r.v2paid);
|
||||
if (vpId == null) continue;
|
||||
const pfId = num(r.pf_party_id) ?? link.get(`vparty:${vpId}`)?.pf ?? null;
|
||||
const id = pfId != null ? `pf:${pfId}` : `vp:${vpId}`;
|
||||
upsert({
|
||||
party_id: id,
|
||||
iso3: r.country_text_id || null,
|
||||
name: r.v2paenname || null,
|
||||
abbrev: r.v2pashname || null,
|
||||
vparty_id: vpId,
|
||||
partyfacts_id: pfId,
|
||||
ches_id: num(r.CHES_ID)
|
||||
});
|
||||
for (const dim of VPARTY_DIMS)
|
||||
if (r[dim] !== undefined && r[dim] !== 'NA') {
|
||||
put({
|
||||
party_id: id,
|
||||
year: Number(r.year),
|
||||
source: 'vparty',
|
||||
dim,
|
||||
value: num(r[dim]),
|
||||
vote_share: num(r.v2pavote)
|
||||
});
|
||||
vpartyRows++;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
note(`V-Party skipped: ${error instanceof Error ? error.message : error}`);
|
||||
}
|
||||
|
||||
// Names for CHES-only parties from Party Facts.
|
||||
for (const r of pf)
|
||||
if (r.dataset_key === 'ches') {
|
||||
const id =
|
||||
num(r.partyfacts_id) != null ? `pf:${r.partyfacts_id}` : `ches:${r.dataset_party_id}`;
|
||||
const p = partiesById.get(id);
|
||||
if (p && !p.name) p.name = r.name_english || r.name || null;
|
||||
}
|
||||
|
||||
const list = [...partiesById.values()];
|
||||
await sql.begin(async (tx) => {
|
||||
await tx`delete from stats.wo_party_position where source in ('ches', 'vparty')`;
|
||||
await tx`delete from stats.wo_party where party_id not like 'wo:%'`;
|
||||
await insertChunks(tx, 'wo_party', list);
|
||||
await insertChunks(tx, 'wo_party_position', [...positions.values()]);
|
||||
// Link curated guide parties to Party Facts where exactly one data party in the same
|
||||
// country has the same abbreviation (case-insensitive).
|
||||
await tx`
|
||||
update stats.wo_party w set partyfacts_id = m.partyfacts_id
|
||||
from (
|
||||
select c.party_id, min(d.partyfacts_id) as partyfacts_id
|
||||
from stats.wo_party c
|
||||
join stats.wo_party d on d.iso3 = c.iso3 and d.party_id not like 'wo:%'
|
||||
and lower(d.abbrev) = lower(c.abbrev) and d.partyfacts_id is not null
|
||||
where c.party_id like 'wo:%'
|
||||
group by c.party_id having count(distinct d.partyfacts_id) = 1
|
||||
) m
|
||||
where w.party_id = m.party_id`;
|
||||
});
|
||||
log(`parties: ${list.length} parties, ${chesRows} CHES + ${vpartyRows} V-Party positions`);
|
||||
return list.length + positions.size;
|
||||
});
|
||||
}
|
||||
123
scripts/stats/remotezip.ts
Normal file
123
scripts/stats/remotezip.ts
Normal file
@@ -0,0 +1,123 @@
|
||||
// Read single entries from a remote zip with HTTP range requests, so a 25 MB archive costs only
|
||||
// its central directory plus the files we need. Falls back to one full download when the server
|
||||
// ignores ranges. Handles stored and deflated entries (no zip64: the archives we read are small).
|
||||
import { inflateRawSync } from 'node:zlib';
|
||||
import { http } from './common';
|
||||
|
||||
export type ZipEntry = {
|
||||
name: string;
|
||||
method: number;
|
||||
compSize: number;
|
||||
size: number;
|
||||
offset: number;
|
||||
};
|
||||
|
||||
export class RemoteZip {
|
||||
private full?: Uint8Array;
|
||||
private constructor(
|
||||
readonly url: string,
|
||||
readonly size: number,
|
||||
readonly entries: ZipEntry[]
|
||||
) {}
|
||||
|
||||
private static async range(url: string, start: number, end: number, size: number) {
|
||||
const res = await http(url, {
|
||||
headers: { range: `bytes=${start}-${end}` },
|
||||
signal: AbortSignal.timeout(1_200_000)
|
||||
});
|
||||
const buf = new Uint8Array(await res.arrayBuffer());
|
||||
if (res.status === 206) return { buf, whole: null as Uint8Array | null };
|
||||
if (buf.length !== size) throw new Error(`short download of ${url}: ${buf.length}/${size}`);
|
||||
return { buf: buf.subarray(start, end + 1), whole: buf };
|
||||
}
|
||||
|
||||
static async open(url: string): Promise<RemoteZip> {
|
||||
const head = await http(url, { method: 'HEAD' });
|
||||
const size = Number(head.headers.get('content-length'));
|
||||
if (!size) throw new Error(`no content-length for ${url}`);
|
||||
const tailLen = Math.min(size, 66000);
|
||||
const tail = await RemoteZip.range(url, size - tailLen, size - 1, size);
|
||||
const t = tail.buf;
|
||||
const v = new DataView(t.buffer, t.byteOffset, t.byteLength);
|
||||
let eocd = -1;
|
||||
for (let i = t.length - 22; i >= 0; i--)
|
||||
if (v.getUint32(i, true) === 0x06054b50) {
|
||||
eocd = i;
|
||||
break;
|
||||
}
|
||||
if (eocd < 0) throw new Error(`not a zip: ${url}`);
|
||||
const cdSize = v.getUint32(eocd + 12, true);
|
||||
const cdOffset = v.getUint32(eocd + 16, true);
|
||||
const cd = tail.whole
|
||||
? tail.whole.subarray(cdOffset, cdOffset + cdSize)
|
||||
: (await RemoteZip.range(url, cdOffset, cdOffset + cdSize - 1, size)).buf;
|
||||
const c = new DataView(cd.buffer, cd.byteOffset, cd.byteLength);
|
||||
const entries: ZipEntry[] = [];
|
||||
const dec = new TextDecoder();
|
||||
for (let p = 0; p + 46 <= cd.length && c.getUint32(p, true) === 0x02014b50;) {
|
||||
const nameLen = c.getUint16(p + 28, true);
|
||||
const extraLen = c.getUint16(p + 30, true);
|
||||
const commentLen = c.getUint16(p + 32, true);
|
||||
entries.push({
|
||||
method: c.getUint16(p + 10, true),
|
||||
compSize: c.getUint32(p + 20, true),
|
||||
size: c.getUint32(p + 24, true),
|
||||
offset: c.getUint32(p + 42, true),
|
||||
name: dec.decode(cd.subarray(p + 46, p + 46 + nameLen))
|
||||
});
|
||||
p += 46 + nameLen + extraLen + commentLen;
|
||||
}
|
||||
const zip = new RemoteZip(url, size, entries);
|
||||
if (tail.whole) zip.full = tail.whole;
|
||||
return zip;
|
||||
}
|
||||
|
||||
find(pattern: RegExp) {
|
||||
return this.entries.filter((e) => pattern.test(e.name) && !e.name.endsWith('/'));
|
||||
}
|
||||
|
||||
async read(entry: ZipEntry): Promise<Uint8Array> {
|
||||
let local: Uint8Array;
|
||||
if (this.full)
|
||||
local = this.full.subarray(entry.offset, entry.offset + 30 + 65535 * 2 + entry.compSize);
|
||||
else
|
||||
local = (
|
||||
await RemoteZip.range(
|
||||
this.url,
|
||||
entry.offset,
|
||||
Math.min(
|
||||
this.size - 1,
|
||||
entry.offset + 30 + 1024 + entry.name.length * 4 + entry.compSize
|
||||
),
|
||||
this.size
|
||||
)
|
||||
).buf;
|
||||
const v = new DataView(local.buffer, local.byteOffset, local.byteLength);
|
||||
if (v.getUint32(0, true) !== 0x04034b50) throw new Error(`bad local header for ${entry.name}`);
|
||||
const start = 30 + v.getUint16(26, true) + v.getUint16(28, true);
|
||||
if (start + entry.compSize > local.length) {
|
||||
// Unusually long local extra field: fetch exactly.
|
||||
const exact = await RemoteZip.range(
|
||||
this.url,
|
||||
entry.offset + start,
|
||||
entry.offset + start + entry.compSize - 1,
|
||||
this.size
|
||||
);
|
||||
return this.inflate(entry, exact.buf);
|
||||
}
|
||||
return this.inflate(entry, local.subarray(start, start + entry.compSize));
|
||||
}
|
||||
|
||||
private inflate(entry: ZipEntry, data: Uint8Array) {
|
||||
if (entry.method === 0) return data;
|
||||
if (entry.method !== 8)
|
||||
throw new Error(`unsupported zip method ${entry.method} (${entry.name})`);
|
||||
const out = new Uint8Array(inflateRawSync(data));
|
||||
if (out.length !== entry.size) throw new Error(`size mismatch for ${entry.name}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
async text(entry: ZipEntry, encoding = 'utf-8') {
|
||||
return new TextDecoder(encoding).decode(await this.read(entry)).replace(/^\uFEFF/, '');
|
||||
}
|
||||
}
|
||||
14
scripts/stats/vparty-csv.py
Normal file
14
scripts/stats/vparty-csv.py
Normal file
@@ -0,0 +1,14 @@
|
||||
# Convert V-Dem's vparty.RData to CSV on stdout (selected columns only). Run with the stats
|
||||
# Python venv (pyreadr + pandas): ~/.local/share/stats-py/bin/python vparty-csv.py file.RData
|
||||
import sys
|
||||
|
||||
import pyreadr
|
||||
|
||||
COLS = [
|
||||
"v2paid", "pf_party_id", "CHES_ID", "country_text_id", "year", "v2paenname", "v2pashname",
|
||||
"v2pavote", "v2paseatshare", "v2xpa_popul", "v2xpa_antiplural", "v2pariglef", "v2paimmig",
|
||||
"v2palgbt", "v2parelig", "v2pawelf", "v2paculsup",
|
||||
]
|
||||
frames = pyreadr.read_r(sys.argv[1])
|
||||
df = next(iter(frames.values()))
|
||||
df[[c for c in COLS if c in df.columns]].to_csv(sys.stdout, index=False)
|
||||
124
scripts/stats/whr.ts
Normal file
124
scripts/stats/whr.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
// A16 World Happiness Report: Figure 2.1 data (life evaluation, 3-year average, and the
|
||||
// explained components), all years 2011–latest. Tries the newest edition first.
|
||||
// © Wellbeing Research Centre / Gallup World Poll data, reuse with attribution.
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { http, insertChunks, log, run, sql } from './common';
|
||||
import { readXlsx, type Cell } from './xlsx';
|
||||
|
||||
const NAME_ISO3: Record<string, string | null> = {
|
||||
'Côte d’Ivoire': 'CIV',
|
||||
"Côte d'Ivoire": 'CIV',
|
||||
'Ivory Coast': 'CIV',
|
||||
'DR Congo': 'COD',
|
||||
'Hong Kong SAR of China': 'HKG',
|
||||
'Hong Kong S.A.R. of China': 'HKG',
|
||||
Kosovo: 'XKX',
|
||||
'Lao PDR': 'LAO',
|
||||
Laos: 'LAO',
|
||||
'Republic of Korea': 'KOR',
|
||||
'South Korea': 'KOR',
|
||||
'Republic of Moldova': 'MDA',
|
||||
'Russian Federation': 'RUS',
|
||||
'State of Palestine': 'PSE',
|
||||
Swaziland: 'SWZ',
|
||||
Eswatini: 'SWZ',
|
||||
'Taiwan Province of China': 'TWN',
|
||||
Türkiye: 'TUR',
|
||||
Turkiye: 'TUR',
|
||||
'Viet Nam': 'VNM',
|
||||
'North Cyprus': null,
|
||||
'Somaliland Region': null,
|
||||
Czechia: 'CZE',
|
||||
'Congo (Brazzaville)': 'COG',
|
||||
'Congo (Kinshasa)': 'COD'
|
||||
};
|
||||
|
||||
const n = (c: Cell | undefined) =>
|
||||
typeof c === 'number'
|
||||
? c
|
||||
: c == null || c === ''
|
||||
? null
|
||||
: Number.isFinite(Number(c))
|
||||
? Number(c)
|
||||
: null;
|
||||
|
||||
export async function whr(opts: { job: string }) {
|
||||
await run('whr', opts.job, async (note) => {
|
||||
const yy = new Date().getFullYear() % 100;
|
||||
let file: string | null = null;
|
||||
for (const e of [yy + 1, yy, yy - 1]) {
|
||||
try {
|
||||
const res = await http(
|
||||
`https://files.worldhappiness.report/WHR${e}_Data_Figure_2.1.xlsx`,
|
||||
{},
|
||||
1
|
||||
);
|
||||
file = join(tmpdir(), `whr-${process.pid}.xlsx`);
|
||||
await Bun.write(file, await res.arrayBuffer());
|
||||
log(`whr: edition 20${e}`);
|
||||
break;
|
||||
} catch {
|
||||
// try an older edition
|
||||
}
|
||||
}
|
||||
if (!file) throw new Error('no WHR Figure 2.1 file found');
|
||||
const [sheet] = await readXlsx(file);
|
||||
await Bun.file(file).delete();
|
||||
const [header, ...rows] = sheet.rows;
|
||||
const col = (re: RegExp) => header.findIndex((h) => re.test(String(h ?? '')));
|
||||
const c = {
|
||||
year: col(/^Year/),
|
||||
rank: col(/^Rank/),
|
||||
country: col(/^Country/),
|
||||
ladder: col(/^Life evaluation|^Ladder/),
|
||||
lower: col(/^Lower/),
|
||||
upper: col(/^Upper/),
|
||||
gdp: col(/GDP/),
|
||||
social: col(/Social/),
|
||||
health: col(/Healthy/),
|
||||
freedom: col(/Freedom/),
|
||||
generosity: col(/Generosity/),
|
||||
corruption: col(/corruption/i),
|
||||
dystopia: col(/Dystopia/)
|
||||
};
|
||||
if (c.year < 0 || c.country < 0 || c.ladder < 0)
|
||||
throw new Error(`WHR columns changed: ${header.join('|')}`);
|
||||
const iso = new Map(
|
||||
(await sql`select distinct country, iso3 from stats.owid_energy where length(iso3) = 3`).map(
|
||||
(r) => [r.country as string, r.iso3 as string]
|
||||
)
|
||||
);
|
||||
const unmapped = new Set<string>();
|
||||
const out = rows
|
||||
.filter((r) => n(r[c.year]) && r[c.country])
|
||||
.map((r) => {
|
||||
const country = String(r[c.country]);
|
||||
const iso3 = country in NAME_ISO3 ? NAME_ISO3[country] : (iso.get(country) ?? null);
|
||||
if (!iso3 && !(country in NAME_ISO3)) unmapped.add(country);
|
||||
const v = (k: keyof typeof c) => (c[k] >= 0 ? n(r[c[k]]) : null);
|
||||
return {
|
||||
year: v('year')!,
|
||||
country,
|
||||
iso3,
|
||||
rank: v('rank'),
|
||||
ladder: v('ladder'),
|
||||
lower: v('lower'),
|
||||
upper: v('upper'),
|
||||
gdp: v('gdp'),
|
||||
social: v('social'),
|
||||
health: v('health'),
|
||||
freedom: v('freedom'),
|
||||
generosity: v('generosity'),
|
||||
corruption: v('corruption'),
|
||||
dystopia: v('dystopia')
|
||||
};
|
||||
});
|
||||
if (unmapped.size) note(`no ISO3 for: ${[...unmapped].join(', ')}`);
|
||||
await sql.begin(async (tx) => {
|
||||
await tx`delete from stats.whr_score`;
|
||||
await insertChunks(tx, 'whr_score', out);
|
||||
});
|
||||
return out.length;
|
||||
});
|
||||
}
|
||||
195
scripts/stats/wikipolls.ts
Normal file
195
scripts/stats/wikipolls.ts
Normal file
@@ -0,0 +1,195 @@
|
||||
// A16 Wikipedia poll tables (CC BY-SA 4.0). For each series we fetch the rendered page via the
|
||||
// MediaWiki API, keep the wikitext snapshot per revision (stats.op_wiki_snapshot), and upsert
|
||||
// one row per poll and option into stats.op_poll. Polls dropped from the page stay in the DB.
|
||||
import { getJson, insertChunks, log, run, sql } from './common';
|
||||
import { fieldworkEnd, findTables, firstNumber, tableGrid, type Cell } from './htmltable';
|
||||
|
||||
const API = 'https://en.wikipedia.org/w/api.php';
|
||||
|
||||
type Series = {
|
||||
series: string;
|
||||
page: string;
|
||||
/** Header labels that identify the table(s) to read. */
|
||||
match: string[];
|
||||
/** How many matching tables to read (newest first on these pages). */
|
||||
tables: number;
|
||||
pollster: RegExp;
|
||||
date: RegExp;
|
||||
sample?: RegExp;
|
||||
skip: RegExp;
|
||||
};
|
||||
|
||||
export const WIKI_SERIES: Series[] = [
|
||||
{
|
||||
series: 'nl-tk-seats',
|
||||
page: 'Opinion_polling_for_the_next_Dutch_general_election',
|
||||
match: ['Polling firm', 'Fieldwork date'],
|
||||
tables: 2,
|
||||
pollster: /^Polling firm/i,
|
||||
date: /^Fieldwork/i,
|
||||
sample: /^Sample/i,
|
||||
skip: /^(Lead|Ref|Ref\.)$/i
|
||||
},
|
||||
{
|
||||
series: 'us-trump-approval-aggregate',
|
||||
page: 'Opinion_polling_on_the_second_Trump_presidency',
|
||||
match: ['Aggregator', 'Approve'],
|
||||
tables: 1,
|
||||
pollster: /^Aggregator/i,
|
||||
date: /^Updated/i,
|
||||
skip: /^(Lead|Net)$/i
|
||||
},
|
||||
{
|
||||
series: 'us-trump-approval',
|
||||
page: 'Opinion_polling_on_the_second_Trump_presidency',
|
||||
match: ['Poll source', 'Approve'],
|
||||
tables: 13,
|
||||
pollster: /^Poll source/i,
|
||||
date: /^Date/i,
|
||||
sample: /^Sample/i,
|
||||
skip: /^(MoE|Net|Lead)$/i
|
||||
},
|
||||
{
|
||||
series: 'kr-lee-approval',
|
||||
page: 'Opinion_polling_on_the_Lee_Jae_Myung_presidency',
|
||||
match: ['Polling firm', 'Approve'],
|
||||
tables: 2,
|
||||
pollster: /^Polling firm/i,
|
||||
date: /^Fieldwork/i,
|
||||
sample: /^Sample/i,
|
||||
skip: /^(Margin|Net|Lead)/i
|
||||
},
|
||||
{
|
||||
series: 'kr-party',
|
||||
page: 'Opinion_polling_for_the_2028_South_Korean_legislative_election',
|
||||
match: ['Polling firm', 'DPK'],
|
||||
tables: 2,
|
||||
pollster: /^Polling firm/i,
|
||||
date: /^Fieldwork/i,
|
||||
sample: /^Sample/i,
|
||||
skip: /^(Margin|Lead)/i
|
||||
}
|
||||
];
|
||||
|
||||
type Poll = {
|
||||
series: string;
|
||||
poll_key: string;
|
||||
fieldwork_end: string;
|
||||
pollster: string;
|
||||
sample: number | null;
|
||||
option: string;
|
||||
value: number;
|
||||
seen_at: Date;
|
||||
};
|
||||
|
||||
export function pollsFromHtml(
|
||||
s: Series,
|
||||
html: string,
|
||||
today = new Date()
|
||||
): Omit<Poll, 'seen_at'>[] {
|
||||
const out: Omit<Poll, 'seen_at'>[] = [];
|
||||
const tables = findTables(html)
|
||||
.map((t) => ({
|
||||
g: tableGrid(t.html),
|
||||
year: Number(t.heading.match(/\b(19|20)\d{2}\b/)?.[0] ?? today.getFullYear())
|
||||
}))
|
||||
.filter(({ g }) => {
|
||||
const head = (g[0] ?? []).map((c) => c.text);
|
||||
return s.match.every((m) => head.some((h) => h.startsWith(m)));
|
||||
})
|
||||
.slice(0, s.tables);
|
||||
const limit = new Date(today.getTime() + 7 * 864e5).toISOString().slice(0, 10);
|
||||
for (const { g, year } of tables) {
|
||||
const head = g[0].map((c) => c.text.replace(/\s*\/\s*/g, '/'));
|
||||
const col = (re: RegExp) => head.findIndex((h) => re.test(h));
|
||||
const [cPoll, cDate, cSample] = [col(s.pollster), col(s.date), s.sample ? col(s.sample) : -1];
|
||||
const options = head
|
||||
.map((h, i) => ({ h, i }))
|
||||
.filter(
|
||||
({ h, i }) =>
|
||||
i !== cPoll &&
|
||||
i !== cDate &&
|
||||
i !== cSample &&
|
||||
h &&
|
||||
!s.skip.test(h) &&
|
||||
head.indexOf(h) === i
|
||||
);
|
||||
for (const row of g.slice(1)) {
|
||||
if (row.every((c) => c.header) || row.length < head.length - 2) continue;
|
||||
const dateCell: Cell | undefined = row[cDate];
|
||||
if (!dateCell) continue;
|
||||
let end = fieldworkEnd(dateCell.text, dateCell.sort, year);
|
||||
if (end && end > limit) end = `${Number(end.slice(0, 4)) - 1}${end.slice(4)}`;
|
||||
if (!end) continue;
|
||||
const pollster = row[cPoll]?.text ?? '';
|
||||
if (!pollster || pollster === dateCell.text) continue;
|
||||
const sample = cSample >= 0 ? firstNumber(row[cSample]?.text ?? '') : null;
|
||||
const key = `${end}|${pollster}|${sample ?? ''}`;
|
||||
for (const { h, i } of options) {
|
||||
const v = firstNumber(row[i]?.text ?? '');
|
||||
if (v != null && !row[i]?.header)
|
||||
out.push({
|
||||
series: s.series,
|
||||
poll_key: key,
|
||||
fieldwork_end: end,
|
||||
pollster,
|
||||
sample,
|
||||
option: h,
|
||||
value: v
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Same poll twice (repeated across tables): keep the first.
|
||||
const seen = new Set<string>();
|
||||
return out.filter((p) => {
|
||||
const k = `${p.poll_key}|${p.option}`;
|
||||
if (seen.has(k)) return false;
|
||||
seen.add(k);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
export async function wikipolls(opts: { job: string }) {
|
||||
await run('wikipedia', opts.job, async (note) => {
|
||||
const pages = new Map<string, { revid: number; html: string; wikitext: string }>();
|
||||
let rows = 0;
|
||||
for (const s of WIKI_SERIES) {
|
||||
let page = pages.get(s.page);
|
||||
if (!page) {
|
||||
const url = `${API}?action=parse&page=${encodeURIComponent(s.page)}&prop=text|wikitext|revid&redirects=1&format=json&formatversion=2`;
|
||||
const res = await getJson<{
|
||||
parse?: { revid: number; text: string; wikitext: string };
|
||||
error?: { info: string };
|
||||
}>(url);
|
||||
if (!res.parse) {
|
||||
note(`${s.page}: ${res.error?.info ?? 'no parse result'}`);
|
||||
continue;
|
||||
}
|
||||
page = { revid: res.parse.revid, html: res.parse.text, wikitext: res.parse.wikitext };
|
||||
pages.set(s.page, page);
|
||||
await sql`
|
||||
insert into stats.op_wiki_snapshot (page, revid, fetched_at, wikitext)
|
||||
values (${s.page}, ${page.revid}, now(), ${page.wikitext})
|
||||
on conflict (page, revid) do nothing`;
|
||||
}
|
||||
const polls = pollsFromHtml(s, page.html);
|
||||
if (!polls.length) {
|
||||
note(`${s.series}: no polls parsed from ${s.page} rev ${page.revid}`);
|
||||
continue;
|
||||
}
|
||||
const now = new Date();
|
||||
await sql.begin(async (tx) => {
|
||||
await insertChunks(
|
||||
tx,
|
||||
'op_poll',
|
||||
polls.map((p) => ({ ...p, seen_at: now })),
|
||||
'on conflict (series, poll_key, option) do update set value = excluded.value, seen_at = excluded.seen_at'
|
||||
);
|
||||
});
|
||||
log(`wikipolls ${s.series}: ${polls.length} values`);
|
||||
rows += polls.length;
|
||||
}
|
||||
return rows;
|
||||
});
|
||||
}
|
||||
1677
scripts/stats/world-opinions-seed.json
Normal file
1677
scripts/stats/world-opinions-seed.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -467,3 +467,280 @@ export const statsCwCbcsBop = statsSchema.table(
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.period, t.item] })]
|
||||
);
|
||||
|
||||
// ── Step 3: elections (A14) ──────────────────────────────────────────────────────────────
|
||||
|
||||
/** One election: NL Tweede Kamer (Kiesraad, CBS 37278) or US presidential (MIT Election Lab). */
|
||||
export const statsElecElection = statsSchema.table('elec_election', {
|
||||
/** NL-TK-2025, US-PRES-2024, … */
|
||||
electionId: text('election_id').primaryKey(),
|
||||
/** ISO 3166-1 alpha-3. */
|
||||
country: char('country', { length: 3 }).notNull(),
|
||||
/** TK (Tweede Kamer), PRES (US president). */
|
||||
kind: text('kind').notNull(),
|
||||
date: date('date', { mode: 'string' }).notNull(),
|
||||
name: text('name').notNull(),
|
||||
sourceId: text('source_id').notNull(),
|
||||
totalSeats: integer('total_seats')
|
||||
});
|
||||
|
||||
export const statsElecResult = statsSchema.table(
|
||||
'elec_result',
|
||||
{
|
||||
electionId: text('election_id').notNull(),
|
||||
/** national, province (kieskring not used), muni, station, state. */
|
||||
level: text('level').notNull(),
|
||||
/** NL / GM0513 / GM0513-SB101 / TX … */
|
||||
areaCode: text('area_code').notNull(),
|
||||
areaName: text('area_name'),
|
||||
/** List name as the source spells it (normalised to a short label on the page). */
|
||||
party: text('party').notNull(),
|
||||
votes: bigint('votes', { mode: 'number' }),
|
||||
seats: integer('seats')
|
||||
},
|
||||
(t) => [
|
||||
primaryKey({ columns: [t.electionId, t.level, t.areaCode, t.party] }),
|
||||
index('elec_result_area_idx').on(t.level, t.areaCode)
|
||||
]
|
||||
);
|
||||
|
||||
export const statsElecTurnout = statsSchema.table(
|
||||
'elec_turnout',
|
||||
{
|
||||
electionId: text('election_id').notNull(),
|
||||
level: text('level').notNull(),
|
||||
areaCode: text('area_code').notNull(),
|
||||
areaName: text('area_name'),
|
||||
/** Persons entitled to vote (kiesgerechtigden). */
|
||||
electorate: bigint('electorate', { mode: 'number' }),
|
||||
/** Ballots cast / counted (opkomst). */
|
||||
ballots: bigint('ballots', { mode: 'number' }),
|
||||
valid: bigint('valid', { mode: 'number' }),
|
||||
blank: bigint('blank', { mode: 'number' }),
|
||||
invalid: bigint('invalid', { mode: 'number' })
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.electionId, t.level, t.areaCode] })]
|
||||
);
|
||||
|
||||
// ── Step 3: world opinions (§4, A15) ─────────────────────────────────────────────────────
|
||||
// The /projects/world-opinions guide reads these. Curated rows (seeded from the former
|
||||
// src/lib/world-opinions/data.ts) keep Stan's text exactly; collectors add CHES / V-Party data.
|
||||
|
||||
export const statsWoCountry = statsSchema.table('wo_country', {
|
||||
iso3: char('iso3', { length: 3 }).primaryKey(),
|
||||
/** ISO 3166-1 alpha-2, lower case (the guide's own id). */
|
||||
id: text('id').notNull().unique(),
|
||||
/** ISO 3166-1 numeric as a zero-padded string (world-atlas shape id). */
|
||||
isoNum: text('iso_num').notNull(),
|
||||
name: text('name').notNull(),
|
||||
/** One of the 11 RegionId keys in src/lib/world-opinions/data.ts. */
|
||||
region: text('region').notNull(),
|
||||
/** left | market | western */
|
||||
liberalMeaning: text('liberal_meaning'),
|
||||
/** economy | culture | history | none */
|
||||
mainDivide: text('main_divide'),
|
||||
note: text('note'),
|
||||
/** Country profile {center, fights[], timeline[{year,label}], blindSpots[]}, or null. */
|
||||
profile: jsonb('profile'),
|
||||
sort: integer('sort').notNull()
|
||||
});
|
||||
|
||||
/** Inglehart–Welzel map positions; the guide's hand-placed values are wave 'curated'. */
|
||||
export const statsWoCulturalMap = statsSchema.table(
|
||||
'wo_cultural_map',
|
||||
{
|
||||
iso3: char('iso3', { length: 3 }).notNull(),
|
||||
wave: text('wave').notNull(),
|
||||
yearFrom: integer('year_from'),
|
||||
yearTo: integer('year_to'),
|
||||
/** y axis: traditional (-) ↔ secular-rational (+). */
|
||||
tradSecular: doublePrecision('trad_secular'),
|
||||
/** x axis: survival (-) ↔ self-expression (+). */
|
||||
survSelfexp: doublePrecision('surv_selfexp'),
|
||||
n: integer('n'),
|
||||
source: text('source')
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.iso3, t.wave] })]
|
||||
);
|
||||
|
||||
/** WVS/EVS/Pew item aggregates (Tier C manual loads; empty until then). */
|
||||
export const statsWoValueScore = statsSchema.table(
|
||||
'wo_value_score',
|
||||
{
|
||||
iso3: char('iso3', { length: 3 }).notNull(),
|
||||
wave: text('wave').notNull(),
|
||||
itemCode: text('item_code').notNull(),
|
||||
itemLabel: text('item_label'),
|
||||
stat: text('stat').notNull(),
|
||||
value: doublePrecision('value'),
|
||||
n: integer('n')
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.iso3, t.wave, t.itemCode, t.stat] })]
|
||||
);
|
||||
|
||||
export const statsWoNarrative = statsSchema.table('wo_narrative', {
|
||||
sort: integer('sort').primaryKey(),
|
||||
event: text('event').notNull(),
|
||||
years: text('years').notNull(),
|
||||
/** [{country (alpha-2), story}] in display order. */
|
||||
views: jsonb('views').notNull()
|
||||
});
|
||||
|
||||
export const statsWoSource = statsSchema.table('wo_source', {
|
||||
sort: integer('sort').primaryKey(),
|
||||
name: text('name').notNull(),
|
||||
href: text('href').notNull()
|
||||
});
|
||||
|
||||
/**
|
||||
* Parties. Curated guide parties: party_id `wo:<cc>:<short>`; data-driven parties:
|
||||
* `pf:<partyfacts_id>` (or `ches:<id>` / `vp:<id>` when Party Facts has no link).
|
||||
*/
|
||||
export const statsWoParty = statsSchema.table(
|
||||
'wo_party',
|
||||
{
|
||||
partyId: text('party_id').primaryKey(),
|
||||
iso3: char('iso3', { length: 3 }),
|
||||
name: text('name'),
|
||||
abbrev: text('abbrev'),
|
||||
family: text('family'),
|
||||
chesId: integer('ches_id'),
|
||||
manifestoId: integer('manifesto_id'),
|
||||
vpartyId: integer('vparty_id'),
|
||||
partyfactsId: integer('partyfacts_id'),
|
||||
/** Display order of curated parties (null for data-driven rows). */
|
||||
sort: integer('sort')
|
||||
},
|
||||
(t) => [index('wo_party_iso3_idx').on(t.iso3)]
|
||||
);
|
||||
|
||||
export const statsWoPartyPosition = statsSchema.table(
|
||||
'wo_party_position',
|
||||
{
|
||||
partyId: text('party_id').notNull(),
|
||||
year: integer('year').notNull(),
|
||||
/** ches, vparty */
|
||||
source: text('source').notNull(),
|
||||
/** lrgen, lrecon, galtan, eu_position, v2xpa_popul, v2pariglef, … */
|
||||
dim: text('dim').notNull(),
|
||||
value: doublePrecision('value'),
|
||||
voteShare: doublePrecision('vote_share')
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.partyId, t.year, t.source, t.dim] })]
|
||||
);
|
||||
|
||||
/** Stan's curated shared-compass placements (x = economic 0–10, y = social 0–10). */
|
||||
export const statsWoCompassOverride = statsSchema.table('wo_compass_override', {
|
||||
partyId: text('party_id').primaryKey(),
|
||||
x: doublePrecision('x').notNull(),
|
||||
y: doublePrecision('y').notNull(),
|
||||
note: text('note')
|
||||
});
|
||||
|
||||
// ── Step 3: opinion feeds (A16) ──────────────────────────────────────────────────────────
|
||||
|
||||
/** Raw Wikipedia wikitext of the poll pages, one row per revision seen. */
|
||||
export const statsOpWikiSnapshot = statsSchema.table(
|
||||
'op_wiki_snapshot',
|
||||
{
|
||||
page: text('page').notNull(),
|
||||
revid: bigint('revid', { mode: 'number' }).notNull(),
|
||||
fetchedAt: timestamptz('fetched_at').notNull().defaultNow(),
|
||||
wikitext: text('wikitext').notNull()
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.page, t.revid] })]
|
||||
);
|
||||
|
||||
/** Parsed poll rows (long format): one value per poll × option. */
|
||||
export const statsOpPoll = statsSchema.table(
|
||||
'op_poll',
|
||||
{
|
||||
/** nl-tk (seats), us-approval, us-approval-avg, us-generic, kr-approval, kr-party. */
|
||||
series: text('series').notNull(),
|
||||
/** `${fieldwork_end}|${pollster}|${n}` (n = order within that pollster and date). */
|
||||
pollKey: text('poll_key').notNull(),
|
||||
fieldworkEnd: date('fieldwork_end', { mode: 'string' }).notNull(),
|
||||
pollster: text('pollster').notNull(),
|
||||
sample: integer('sample'),
|
||||
option: text('option').notNull(),
|
||||
value: doublePrecision('value'),
|
||||
seenAt: timestamptz('seen_at').notNull().defaultNow()
|
||||
},
|
||||
(t) => [
|
||||
primaryKey({ columns: [t.series, t.pollKey, t.option] }),
|
||||
index('op_poll_series_date_idx').on(t.series, t.fieldworkEnd)
|
||||
]
|
||||
);
|
||||
|
||||
/** Standard Eurobarometer volume A: most-important-issue and trust questions per country. */
|
||||
export const statsEbItem = statsSchema.table(
|
||||
'eb_item',
|
||||
{
|
||||
/** 104.1 etc. */
|
||||
wave: text('wave').notNull(),
|
||||
fieldworkEnd: date('fieldwork_end', { mode: 'string' }),
|
||||
/** mii_country, mii_personal, mii_eu, trust */
|
||||
topic: text('topic').notNull(),
|
||||
/** Sheet name in the wave's file (QA3, QA6_9, …). */
|
||||
qcode: text('qcode').notNull(),
|
||||
/** Answer (issue) for MII; institution for trust. */
|
||||
item: text('item').notNull(),
|
||||
/** Eurobarometer country code (EU27, BE, EL, …). */
|
||||
country: text('country').notNull(),
|
||||
/** Share of respondents, 0–1. */
|
||||
share: doublePrecision('share')
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.wave, t.topic, t.item, t.country] })]
|
||||
);
|
||||
|
||||
/** World Happiness Report: life evaluation (3-year average) and its explained parts. */
|
||||
export const statsWhrScore = statsSchema.table(
|
||||
'whr_score',
|
||||
{
|
||||
year: integer('year').notNull(),
|
||||
country: text('country').notNull(),
|
||||
iso3: char('iso3', { length: 3 }),
|
||||
rank: integer('rank'),
|
||||
ladder: doublePrecision('ladder'),
|
||||
lower: doublePrecision('lower'),
|
||||
upper: doublePrecision('upper'),
|
||||
gdp: doublePrecision('gdp'),
|
||||
social: doublePrecision('social'),
|
||||
health: doublePrecision('health'),
|
||||
freedom: doublePrecision('freedom'),
|
||||
generosity: doublePrecision('generosity'),
|
||||
corruption: doublePrecision('corruption'),
|
||||
dystopia: doublePrecision('dystopia')
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.year, t.country] })]
|
||||
);
|
||||
|
||||
/** Google Trends daily trending searches (RSS snapshot). */
|
||||
export const statsGtTrending = statsSchema.table(
|
||||
'gt_trending',
|
||||
{
|
||||
geo: text('geo').notNull(),
|
||||
date: date('date', { mode: 'string' }).notNull(),
|
||||
rank: integer('rank').notNull(),
|
||||
query: text('query').notNull(),
|
||||
approxTraffic: integer('approx_traffic'),
|
||||
pubDate: timestamptz('pub_date'),
|
||||
/** First related news headline + source, for context. */
|
||||
news: text('news')
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.geo, t.date, t.query] })]
|
||||
);
|
||||
|
||||
/** GDELT DOC timelinevol: share of monitored news coverage per theme and source country. */
|
||||
export const statsGdeltThemeVol = statsSchema.table(
|
||||
'gdelt_theme_vol',
|
||||
{
|
||||
date: date('date', { mode: 'string' }).notNull(),
|
||||
/** ISO 3166-1 alpha-3 of the source country. */
|
||||
country: char('country', { length: 3 }).notNull(),
|
||||
theme: text('theme').notNull(),
|
||||
/** Percentage of all monitored articles (daily mean of GDELT's volume intensity). */
|
||||
volPct: doublePrecision('vol_pct')
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.date, t.country, t.theme] })]
|
||||
);
|
||||
|
||||
76
src/lib/server/world-opinions.ts
Normal file
76
src/lib/server/world-opinions.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
import { client } from '#lib/server/db/index.js';
|
||||
import { cached } from '#lib/server/stats.js';
|
||||
import type {
|
||||
Country,
|
||||
Divide,
|
||||
LiberalMeaning,
|
||||
Narrative,
|
||||
Party,
|
||||
Profile,
|
||||
RegionId,
|
||||
Source,
|
||||
WorldOpinionsData
|
||||
} from '#lib/world-opinions/data.js';
|
||||
|
||||
// /projects/world-opinions content from stats.wo_* (seeded from the former data.ts). Rebuilds
|
||||
// the exact shapes the components used, leaving optional keys out when the column is null.
|
||||
export function loadWorldOpinions(): Promise<WorldOpinionsData> {
|
||||
return cached('world-opinions', async () => {
|
||||
const [countryRows, partyRows, narrativeRows, sourceRows] = await Promise.all([
|
||||
client`
|
||||
select c.id, c.iso_num, c.name, c.region, c.liberal_meaning, c.main_divide, c.note,
|
||||
c.profile, m.surv_selfexp, m.trad_secular
|
||||
from stats.wo_country c
|
||||
left join stats.wo_cultural_map m on m.iso3 = c.iso3 and m.wave = 'curated'
|
||||
order by c.sort`,
|
||||
client`
|
||||
select c.id as country, p.abbrev, p.name, o.x, o.y, o.note
|
||||
from stats.wo_party p
|
||||
join stats.wo_compass_override o using (party_id)
|
||||
join stats.wo_country c on c.iso3 = p.iso3
|
||||
where p.sort is not null
|
||||
order by p.sort`,
|
||||
client`select event, years, views from stats.wo_narrative order by sort`,
|
||||
client`select name, href from stats.wo_source order by sort`
|
||||
]);
|
||||
const countries: Country[] = [];
|
||||
const profiles: Profile[] = [];
|
||||
for (const r of countryRows) {
|
||||
const country: Country = {
|
||||
id: r.id,
|
||||
iso: r.iso_num,
|
||||
name: r.name,
|
||||
region: r.region as RegionId
|
||||
};
|
||||
if (r.surv_selfexp != null && r.trad_secular != null)
|
||||
country.values = { x: r.surv_selfexp, y: r.trad_secular };
|
||||
if (r.liberal_meaning) country.liberal = r.liberal_meaning as LiberalMeaning;
|
||||
if (r.main_divide) country.divide = r.main_divide as Divide;
|
||||
if (r.note) country.note = r.note;
|
||||
countries.push(country);
|
||||
if (r.profile) profiles.push({ id: r.id, ...(r.profile as Omit<Profile, 'id'>) });
|
||||
}
|
||||
const parties: Party[] = partyRows.map((r) => {
|
||||
const party: Party = {
|
||||
country: r.country,
|
||||
short: r.abbrev,
|
||||
name: r.name,
|
||||
econ: r.x,
|
||||
social: r.y
|
||||
};
|
||||
if (r.note) party.note = r.note;
|
||||
return party;
|
||||
});
|
||||
return {
|
||||
countries,
|
||||
parties,
|
||||
profiles,
|
||||
narratives: narrativeRows.map((r) => ({
|
||||
event: r.event,
|
||||
years: r.years,
|
||||
views: r.views as Narrative['views']
|
||||
})),
|
||||
sources: sourceRows.map((r) => ({ name: r.name, href: r.href }) as Source)
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
<script lang="ts">
|
||||
import EyeOff from 'lucide-svelte/icons/eye-off';
|
||||
import Swords from 'lucide-svelte/icons/swords';
|
||||
import { countryById, divides, liberalMeanings, profileById, regions } from './data';
|
||||
import { divides, liberalMeanings, regions, worldOpinions } from './data';
|
||||
|
||||
const { countryById, profileById } = worldOpinions();
|
||||
|
||||
let { id }: { id: string } = $props();
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
<script lang="ts">
|
||||
import { countries, regions, type RegionId } from './data';
|
||||
import { regions, worldOpinions, type RegionId } from './data';
|
||||
import { Button } from '#lib/components/ui/button/index.js';
|
||||
|
||||
const { countries } = worldOpinions();
|
||||
|
||||
const culturalMap = countries.flatMap((country) =>
|
||||
country.values ? [{ ...country, x: country.values.x, y: country.values.y }] : []
|
||||
);
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
<script lang="ts">
|
||||
import { countryById, parties, partyCountries } from './data';
|
||||
import { worldOpinions } from './data';
|
||||
import { Button } from '#lib/components/ui/button/index.js';
|
||||
|
||||
const { countryById, parties, partyCountries } = worldOpinions();
|
||||
|
||||
let { focus = null }: { focus?: string | null } = $props();
|
||||
|
||||
const MAX = 3;
|
||||
|
||||
@@ -31,7 +31,9 @@
|
||||
</script>
|
||||
|
||||
<script lang="ts">
|
||||
import { countries, divides, liberalMeanings, type Country } from './data';
|
||||
import { divides, liberalMeanings, worldOpinions, type Country } from './data';
|
||||
|
||||
const { countries } = worldOpinions();
|
||||
|
||||
let {
|
||||
mode,
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import world from 'world-atlas/countries-110m.json';
|
||||
import { countries, countryById, narratives, parties, partyCountries, profiles } from './data';
|
||||
import seed from '../../../scripts/stats/world-opinions-seed.json';
|
||||
import { buildWorldOpinions, type WorldOpinionsData } from './data';
|
||||
|
||||
describe('world opinions data', () => {
|
||||
// The guide's content lives in stats.wo_*; this checks the snapshot the stats_wo_seed migration
|
||||
// was generated from (scripts/stats/world-opinions-seed.json).
|
||||
const { countries, countryById, narratives, parties, partyCountries, profiles } =
|
||||
buildWorldOpinions(seed as WorldOpinionsData);
|
||||
|
||||
describe('world opinions seed', () => {
|
||||
it('has unique country ids and ISO codes', () => {
|
||||
expect(new Set(countries.map((country) => country.id)).size).toBe(countries.length);
|
||||
expect(new Set(countries.map((country) => country.iso)).size).toBe(countries.length);
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
// World Opinions: a hand-curated orientation guide, not a dataset of record.
|
||||
// Positions are approximate and meant for comparing countries/parties relative
|
||||
// to each other. Sources are listed in `sources` at the bottom of this file.
|
||||
// to each other. The curated content (countries, party placements, profiles,
|
||||
// narratives, sources) lives in the `stats.wo_*` tables (seeded by the
|
||||
// stats_wo_seed migration) and is loaded by +page.server.ts; this file keeps
|
||||
// the types, the label dictionaries and the lookups the components share.
|
||||
import { getContext, setContext } from 'svelte';
|
||||
|
||||
export type RegionId =
|
||||
| 'north-america'
|
||||
@@ -63,483 +67,6 @@ export type Country = {
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export const countries: Country[] = [
|
||||
{
|
||||
id: 'se',
|
||||
iso: '752',
|
||||
name: 'Sweden',
|
||||
region: 'nordics',
|
||||
values: { x: 2.4, y: 1.8 },
|
||||
liberal: 'market',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'dk',
|
||||
iso: '208',
|
||||
name: 'Denmark',
|
||||
region: 'nordics',
|
||||
values: { x: 2.3, y: 1.2 },
|
||||
liberal: 'market',
|
||||
divide: 'economy',
|
||||
note: '"Venstre" (literally "Left") is the liberal, center-right party'
|
||||
},
|
||||
{
|
||||
id: 'no',
|
||||
iso: '578',
|
||||
name: 'Norway',
|
||||
region: 'nordics',
|
||||
values: { x: 2.1, y: 1.4 },
|
||||
liberal: 'market',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'fi',
|
||||
iso: '246',
|
||||
name: 'Finland',
|
||||
region: 'nordics',
|
||||
values: { x: 1.6, y: 1.0 },
|
||||
liberal: 'market',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'nl',
|
||||
iso: '528',
|
||||
name: 'Netherlands',
|
||||
region: 'western-europe',
|
||||
values: { x: 2.0, y: 1.1 },
|
||||
liberal: 'market',
|
||||
divide: 'economy',
|
||||
note: 'Liberal VVD is the main center-right party'
|
||||
},
|
||||
{
|
||||
id: 'be',
|
||||
iso: '056',
|
||||
name: 'Belgium',
|
||||
region: 'western-europe',
|
||||
values: { x: 1.4, y: 0.8 },
|
||||
liberal: 'market',
|
||||
divide: 'economy',
|
||||
note: 'Language (Flemish vs French-speaking) splits every party family'
|
||||
},
|
||||
{
|
||||
id: 'de',
|
||||
iso: '276',
|
||||
name: 'Germany',
|
||||
region: 'western-europe',
|
||||
values: { x: 1.3, y: 1.4 },
|
||||
liberal: 'market',
|
||||
divide: 'economy',
|
||||
note: 'Liberal FDP = pro-market, low taxes'
|
||||
},
|
||||
{
|
||||
id: 'at',
|
||||
iso: '040',
|
||||
name: 'Austria',
|
||||
region: 'western-europe',
|
||||
values: { x: 1.3, y: 0.6 },
|
||||
liberal: 'market',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'ch',
|
||||
iso: '756',
|
||||
name: 'Switzerland',
|
||||
region: 'western-europe',
|
||||
values: { x: 1.9, y: 0.7 },
|
||||
liberal: 'market',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'gb',
|
||||
iso: '826',
|
||||
name: 'United Kingdom',
|
||||
region: 'western-europe',
|
||||
values: { x: 1.8, y: 0.3 },
|
||||
liberal: 'market',
|
||||
divide: 'economy',
|
||||
note: 'Liberal Democrats sit in the center; Brexit cut across old lines'
|
||||
},
|
||||
{
|
||||
id: 'ie',
|
||||
iso: '372',
|
||||
name: 'Ireland',
|
||||
region: 'western-europe',
|
||||
values: { x: 1.4, y: -0.4 },
|
||||
liberal: 'market',
|
||||
divide: 'history',
|
||||
note: 'The two big parties descend from opposite sides of the 1922 civil war'
|
||||
},
|
||||
{
|
||||
id: 'fr',
|
||||
iso: '250',
|
||||
name: 'France',
|
||||
region: 'western-europe',
|
||||
values: { x: 1.1, y: 0.8 },
|
||||
liberal: 'market',
|
||||
divide: 'economy',
|
||||
note: '"Libéral" means pro-market, often used as a criticism'
|
||||
},
|
||||
{
|
||||
id: 'es',
|
||||
iso: '724',
|
||||
name: 'Spain',
|
||||
region: 'western-europe',
|
||||
values: { x: 1.3, y: 0.3 },
|
||||
liberal: 'market',
|
||||
divide: 'economy',
|
||||
note: 'Regional nationalism (Catalonia, Basque Country) is a second axis'
|
||||
},
|
||||
{
|
||||
id: 'pt',
|
||||
iso: '620',
|
||||
name: 'Portugal',
|
||||
region: 'western-europe',
|
||||
values: { x: 0.8, y: -0.3 },
|
||||
liberal: 'market',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'it',
|
||||
iso: '380',
|
||||
name: 'Italy',
|
||||
region: 'western-europe',
|
||||
values: { x: 0.5, y: 0.0 },
|
||||
liberal: 'market',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'gr',
|
||||
iso: '300',
|
||||
name: 'Greece',
|
||||
region: 'eastern-europe',
|
||||
values: { x: 0.2, y: 0.0 },
|
||||
liberal: 'market',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'us',
|
||||
iso: '840',
|
||||
name: 'United States',
|
||||
region: 'north-america',
|
||||
values: { x: 1.3, y: -0.5 },
|
||||
liberal: 'left',
|
||||
divide: 'culture',
|
||||
note: '"Liberal" = Democrat; the Democrats sit near Europe\'s center economically'
|
||||
},
|
||||
{
|
||||
id: 'ca',
|
||||
iso: '124',
|
||||
name: 'Canada',
|
||||
region: 'north-america',
|
||||
values: { x: 1.8, y: 0.0 },
|
||||
liberal: 'left',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'mx',
|
||||
iso: '484',
|
||||
name: 'Mexico',
|
||||
region: 'latin-america',
|
||||
values: { x: 0.6, y: -1.3 },
|
||||
liberal: 'market',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'br',
|
||||
iso: '076',
|
||||
name: 'Brazil',
|
||||
region: 'latin-america',
|
||||
values: { x: 0.1, y: -1.0 },
|
||||
liberal: 'market',
|
||||
divide: 'economy',
|
||||
note: 'Bolsonaro\'s party is literally called "Partido Liberal"'
|
||||
},
|
||||
{
|
||||
id: 'ar',
|
||||
iso: '032',
|
||||
name: 'Argentina',
|
||||
region: 'latin-america',
|
||||
values: { x: 0.7, y: -0.6 },
|
||||
liberal: 'market',
|
||||
divide: 'economy',
|
||||
note: 'President Milei calls himself a "liberal-libertarian"'
|
||||
},
|
||||
{
|
||||
id: 'cl',
|
||||
iso: '152',
|
||||
name: 'Chile',
|
||||
region: 'latin-america',
|
||||
values: { x: 0.8, y: -0.3 },
|
||||
liberal: 'market',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'co',
|
||||
iso: '170',
|
||||
name: 'Colombia',
|
||||
region: 'latin-america',
|
||||
values: { x: 0.5, y: -1.6 },
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'pe',
|
||||
iso: '604',
|
||||
name: 'Peru',
|
||||
region: 'latin-america',
|
||||
values: { x: 0.3, y: -1.1 },
|
||||
liberal: 'market',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 've',
|
||||
iso: '862',
|
||||
name: 'Venezuela',
|
||||
region: 'latin-america',
|
||||
values: { x: 0.4, y: -1.5 },
|
||||
divide: 'none'
|
||||
},
|
||||
{
|
||||
id: 'au',
|
||||
iso: '036',
|
||||
name: 'Australia',
|
||||
region: 'oceania',
|
||||
values: { x: 2.0, y: 0.2 },
|
||||
liberal: 'market',
|
||||
divide: 'economy',
|
||||
note: 'The Liberal Party is the main conservative party'
|
||||
},
|
||||
{
|
||||
id: 'nz',
|
||||
iso: '554',
|
||||
name: 'New Zealand',
|
||||
region: 'oceania',
|
||||
values: { x: 2.2, y: 0.1 },
|
||||
liberal: 'market',
|
||||
divide: 'economy'
|
||||
},
|
||||
{
|
||||
id: 'pl',
|
||||
iso: '616',
|
||||
name: 'Poland',
|
||||
region: 'eastern-europe',
|
||||
values: { x: 0.2, y: -0.6 },
|
||||
liberal: 'western',
|
||||
divide: 'culture'
|
||||
},
|
||||
{
|
||||
id: 'cz',
|
||||
iso: '203',
|
||||
name: 'Czechia',
|
||||
region: 'eastern-europe',
|
||||
values: { x: 0.5, y: 1.2 },
|
||||
liberal: 'western',
|
||||
divide: 'culture'
|
||||
},
|
||||
{
|
||||
id: 'hu',
|
||||
iso: '348',
|
||||
name: 'Hungary',
|
||||
region: 'eastern-europe',
|
||||
values: { x: -0.4, y: 0.4 },
|
||||
liberal: 'western',
|
||||
divide: 'culture',
|
||||
note: 'Orbán calls his model "illiberal democracy"'
|
||||
},
|
||||
{
|
||||
id: 'ro',
|
||||
iso: '642',
|
||||
name: 'Romania',
|
||||
region: 'eastern-europe',
|
||||
values: { x: -0.4, y: -0.6 },
|
||||
liberal: 'western',
|
||||
divide: 'culture'
|
||||
},
|
||||
{
|
||||
id: 'rs',
|
||||
iso: '688',
|
||||
name: 'Serbia',
|
||||
region: 'eastern-europe',
|
||||
values: { x: -0.6, y: 0.1 },
|
||||
liberal: 'western',
|
||||
divide: 'history'
|
||||
},
|
||||
{
|
||||
id: 'ua',
|
||||
iso: '804',
|
||||
name: 'Ukraine',
|
||||
region: 'eastern-europe',
|
||||
values: { x: -0.9, y: 0.2 },
|
||||
divide: 'history',
|
||||
note: 'Politics long split between pro-Western and pro-Russian orientation'
|
||||
},
|
||||
{
|
||||
id: 'ru',
|
||||
iso: '643',
|
||||
name: 'Russia',
|
||||
region: 'eastern-europe',
|
||||
values: { x: -1.0, y: 0.6 },
|
||||
liberal: 'western',
|
||||
divide: 'none',
|
||||
note: '"Liberal" = the pro-Western opposition, mostly in exile or prison'
|
||||
},
|
||||
{
|
||||
id: 'kz',
|
||||
iso: '398',
|
||||
name: 'Kazakhstan',
|
||||
region: 'eastern-europe',
|
||||
values: { x: -0.6, y: 0.4 },
|
||||
divide: 'none'
|
||||
},
|
||||
{
|
||||
id: 'tr',
|
||||
iso: '792',
|
||||
name: 'Turkey',
|
||||
region: 'middle-east',
|
||||
values: { x: -0.6, y: -0.9 },
|
||||
liberal: 'western',
|
||||
divide: 'culture',
|
||||
note: 'Main divide: religious-conservative vs secular'
|
||||
},
|
||||
{
|
||||
id: 'ir',
|
||||
iso: '364',
|
||||
name: 'Iran',
|
||||
region: 'middle-east',
|
||||
values: { x: -0.9, y: -0.6 },
|
||||
liberal: 'western',
|
||||
divide: 'none'
|
||||
},
|
||||
{
|
||||
id: 'iq',
|
||||
iso: '368',
|
||||
name: 'Iraq',
|
||||
region: 'middle-east',
|
||||
values: { x: -1.2, y: -1.3 },
|
||||
divide: 'culture'
|
||||
},
|
||||
{
|
||||
id: 'eg',
|
||||
iso: '818',
|
||||
name: 'Egypt',
|
||||
region: 'middle-east',
|
||||
values: { x: -1.5, y: -1.5 },
|
||||
divide: 'none'
|
||||
},
|
||||
{ id: 'ma', iso: '504', name: 'Morocco', region: 'middle-east', values: { x: -1.2, y: -1.5 } },
|
||||
{ id: 'sa', iso: '682', name: 'Saudi Arabia', region: 'middle-east', divide: 'none' },
|
||||
{ id: 'ng', iso: '566', name: 'Nigeria', region: 'africa', values: { x: -0.3, y: -1.8 } },
|
||||
{ id: 'et', iso: '231', name: 'Ethiopia', region: 'africa', values: { x: -0.9, y: -1.5 } },
|
||||
{ id: 'ke', iso: '404', name: 'Kenya', region: 'africa', values: { x: -0.4, y: -1.5 } },
|
||||
{
|
||||
id: 'za',
|
||||
iso: '710',
|
||||
name: 'South Africa',
|
||||
region: 'africa',
|
||||
values: { x: 0.0, y: -1.2 },
|
||||
divide: 'history',
|
||||
note: 'Politics still shaped by apartheid and the liberation struggle'
|
||||
},
|
||||
{
|
||||
id: 'in',
|
||||
iso: '356',
|
||||
name: 'India',
|
||||
region: 'south-asia',
|
||||
values: { x: -0.5, y: -0.5 },
|
||||
divide: 'culture',
|
||||
note: 'Main divide: Hindu nationalism vs secular pluralism'
|
||||
},
|
||||
{ id: 'pk', iso: '586', name: 'Pakistan', region: 'south-asia', values: { x: -1.0, y: -1.4 } },
|
||||
{
|
||||
id: 'bd',
|
||||
iso: '050',
|
||||
name: 'Bangladesh',
|
||||
region: 'south-asia',
|
||||
values: { x: -1.0, y: -1.1 },
|
||||
divide: 'history'
|
||||
},
|
||||
{
|
||||
id: 'cn',
|
||||
iso: '156',
|
||||
name: 'China',
|
||||
region: 'east-asia',
|
||||
values: { x: -0.9, y: 1.5 },
|
||||
liberal: 'western',
|
||||
divide: 'none',
|
||||
note: '"Liberal" = pro-Western intellectuals, a label the state treats with suspicion'
|
||||
},
|
||||
{
|
||||
id: 'jp',
|
||||
iso: '392',
|
||||
name: 'Japan',
|
||||
region: 'east-asia',
|
||||
values: { x: 0.4, y: 1.8 },
|
||||
liberal: 'market',
|
||||
divide: 'history',
|
||||
note: 'The "Liberal Democratic Party" is the conservative party in power since 1955'
|
||||
},
|
||||
{
|
||||
id: 'kr',
|
||||
iso: '410',
|
||||
name: 'South Korea',
|
||||
region: 'east-asia',
|
||||
values: { x: -0.3, y: 1.2 },
|
||||
liberal: 'left',
|
||||
divide: 'history',
|
||||
note: 'Divide is mostly about North Korea, Japan and the US'
|
||||
},
|
||||
{
|
||||
id: 'tw',
|
||||
iso: '158',
|
||||
name: 'Taiwan',
|
||||
region: 'east-asia',
|
||||
values: { x: 0.0, y: 1.3 },
|
||||
divide: 'history',
|
||||
note: 'Main divide: how close to be with mainland China'
|
||||
},
|
||||
{
|
||||
id: 'vn',
|
||||
iso: '704',
|
||||
name: 'Vietnam',
|
||||
region: 'southeast-asia',
|
||||
values: { x: -0.6, y: 0.5 },
|
||||
divide: 'none'
|
||||
},
|
||||
{
|
||||
id: 'th',
|
||||
iso: '764',
|
||||
name: 'Thailand',
|
||||
region: 'southeast-asia',
|
||||
values: { x: 0.0, y: -0.4 },
|
||||
divide: 'history',
|
||||
note: 'Main divide: monarchy and military vs reformists'
|
||||
},
|
||||
{
|
||||
id: 'my',
|
||||
iso: '458',
|
||||
name: 'Malaysia',
|
||||
region: 'southeast-asia',
|
||||
values: { x: -0.6, y: -0.9 },
|
||||
divide: 'culture'
|
||||
},
|
||||
{
|
||||
id: 'ph',
|
||||
iso: '608',
|
||||
name: 'Philippines',
|
||||
region: 'southeast-asia',
|
||||
values: { x: -0.2, y: -1.4 }
|
||||
},
|
||||
{
|
||||
id: 'id',
|
||||
iso: '360',
|
||||
name: 'Indonesia',
|
||||
region: 'southeast-asia',
|
||||
values: { x: -0.8, y: -1.2 },
|
||||
divide: 'culture'
|
||||
}
|
||||
];
|
||||
|
||||
export const countryById = new Map(countries.map((country) => [country.id, country]));
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Party compass: parties on one shared scale
|
||||
// econ: 0 = far left (state-led, redistributive) … 10 = far right (free market)
|
||||
@@ -557,154 +84,7 @@ export type Party = {
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export const partyCountries = [
|
||||
'us',
|
||||
'nl',
|
||||
'de',
|
||||
'gb',
|
||||
'fr',
|
||||
'se',
|
||||
'pl',
|
||||
'jp',
|
||||
'kr',
|
||||
'in',
|
||||
'br',
|
||||
'ca'
|
||||
];
|
||||
|
||||
export const parties: Party[] = [
|
||||
{
|
||||
country: 'us',
|
||||
short: 'Dem',
|
||||
name: 'Democratic Party',
|
||||
econ: 4.8,
|
||||
social: 3.5,
|
||||
note: 'The US "liberals": near Europe\'s center economically'
|
||||
},
|
||||
{ country: 'us', short: 'GOP', name: 'Republican Party', econ: 7.8, social: 8.0 },
|
||||
|
||||
{ country: 'nl', short: 'GL-PvdA', name: 'GroenLinks-PvdA', econ: 2.5, social: 1.5 },
|
||||
{ country: 'nl', short: 'SP', name: 'Socialistische Partij', econ: 1.5, social: 5.0 },
|
||||
{ country: 'nl', short: 'D66', name: 'Democraten 66', econ: 5.5, social: 1.5 },
|
||||
{ country: 'nl', short: 'CDA', name: 'Christen-Democratisch Appèl', econ: 6.0, social: 6.0 },
|
||||
{
|
||||
country: 'nl',
|
||||
short: 'VVD',
|
||||
name: 'Volkspartij voor Vrijheid en Democratie',
|
||||
econ: 7.5,
|
||||
social: 5.0,
|
||||
note: 'Dutch "liberals": market-liberal, center-right'
|
||||
},
|
||||
{ country: 'nl', short: 'BBB', name: 'BoerBurgerBeweging', econ: 6.0, social: 7.5 },
|
||||
{
|
||||
country: 'nl',
|
||||
short: 'PVV',
|
||||
name: 'Partij voor de Vrijheid',
|
||||
econ: 4.5,
|
||||
social: 9.0,
|
||||
note: 'Radical right on immigration, centrist on welfare'
|
||||
},
|
||||
|
||||
{ country: 'de', short: 'Linke', name: 'Die Linke', econ: 1.0, social: 2.5 },
|
||||
{ country: 'de', short: 'Grüne', name: 'Bündnis 90/Die Grünen', econ: 3.0, social: 1.5 },
|
||||
{ country: 'de', short: 'SPD', name: 'Sozialdemokratische Partei', econ: 3.5, social: 3.5 },
|
||||
{ country: 'de', short: 'BSW', name: 'Bündnis Sahra Wagenknecht', econ: 2.0, social: 7.0 },
|
||||
{
|
||||
country: 'de',
|
||||
short: 'FDP',
|
||||
name: 'Freie Demokratische Partei',
|
||||
econ: 8.5,
|
||||
social: 3.0,
|
||||
note: 'German "liberals": socially open, strongly pro-market'
|
||||
},
|
||||
{ country: 'de', short: 'CDU', name: 'CDU/CSU', econ: 6.5, social: 6.0 },
|
||||
{ country: 'de', short: 'AfD', name: 'Alternative für Deutschland', econ: 7.0, social: 9.5 },
|
||||
|
||||
{ country: 'gb', short: 'Green', name: 'Green Party', econ: 1.5, social: 1.5 },
|
||||
{ country: 'gb', short: 'Lab', name: 'Labour', econ: 3.8, social: 3.5 },
|
||||
{ country: 'gb', short: 'LD', name: 'Liberal Democrats', econ: 5.0, social: 2.5 },
|
||||
{ country: 'gb', short: 'Con', name: 'Conservatives', econ: 7.0, social: 6.5 },
|
||||
{ country: 'gb', short: 'Reform', name: 'Reform UK', econ: 7.5, social: 8.5 },
|
||||
|
||||
{ country: 'fr', short: 'LFI', name: 'La France Insoumise', econ: 1.0, social: 2.5 },
|
||||
{ country: 'fr', short: 'PS', name: 'Parti Socialiste', econ: 3.0, social: 3.0 },
|
||||
{ country: 'fr', short: 'RE', name: 'Renaissance', econ: 6.5, social: 4.0 },
|
||||
{ country: 'fr', short: 'LR', name: 'Les Républicains', econ: 7.5, social: 7.0 },
|
||||
{
|
||||
country: 'fr',
|
||||
short: 'RN',
|
||||
name: 'Rassemblement National',
|
||||
econ: 4.5,
|
||||
social: 9.0,
|
||||
note: 'Economically left of Macron: lower retirement age, protectionism'
|
||||
},
|
||||
|
||||
{ country: 'se', short: 'V', name: 'Vänsterpartiet', econ: 1.0, social: 2.0 },
|
||||
{ country: 'se', short: 'S', name: 'Socialdemokraterna', econ: 3.0, social: 3.5 },
|
||||
{ country: 'se', short: 'C', name: 'Centerpartiet', econ: 7.0, social: 2.5 },
|
||||
{ country: 'se', short: 'M', name: 'Moderaterna', econ: 7.5, social: 5.5 },
|
||||
{ country: 'se', short: 'SD', name: 'Sverigedemokraterna', econ: 5.5, social: 9.0 },
|
||||
|
||||
{ country: 'pl', short: 'Lewica', name: 'Lewica', econ: 2.0, social: 2.0 },
|
||||
{ country: 'pl', short: 'KO', name: 'Koalicja Obywatelska', econ: 6.5, social: 4.0 },
|
||||
{
|
||||
country: 'pl',
|
||||
short: 'PiS',
|
||||
name: 'Prawo i Sprawiedliwość',
|
||||
econ: 3.5,
|
||||
social: 9.0,
|
||||
note: 'Culturally right, economically left (big family benefits)'
|
||||
},
|
||||
{ country: 'pl', short: 'Konf', name: 'Konfederacja', econ: 9.0, social: 9.0 },
|
||||
|
||||
{ country: 'jp', short: 'JCP', name: 'Japanese Communist Party', econ: 1.0, social: 3.0 },
|
||||
{ country: 'jp', short: 'CDP', name: 'Constitutional Democratic Party', econ: 4.0, social: 3.5 },
|
||||
{ country: 'jp', short: 'Komeito', name: 'Komeito', econ: 5.0, social: 5.5 },
|
||||
{
|
||||
country: 'jp',
|
||||
short: 'LDP',
|
||||
name: 'Liberal Democratic Party',
|
||||
econ: 6.0,
|
||||
social: 7.0,
|
||||
note: 'Called "Liberal", governs as conservatives since 1955'
|
||||
},
|
||||
{ country: 'jp', short: 'Ishin', name: 'Nippon Ishin no Kai', econ: 7.5, social: 6.0 },
|
||||
{ country: 'jp', short: 'Sansei', name: 'Sanseitō', econ: 6.0, social: 9.0 },
|
||||
|
||||
{ country: 'kr', short: 'Justice', name: 'Justice Party', econ: 1.5, social: 1.5 },
|
||||
{
|
||||
country: 'kr',
|
||||
short: 'DPK',
|
||||
name: 'Democratic Party of Korea',
|
||||
econ: 4.0,
|
||||
social: 4.5,
|
||||
note: 'The "progressive" camp, cautious on social issues'
|
||||
},
|
||||
{ country: 'kr', short: 'PPP', name: 'People Power Party', econ: 7.0, social: 7.0 },
|
||||
|
||||
{ country: 'in', short: 'INC', name: 'Indian National Congress', econ: 4.5, social: 4.0 },
|
||||
{
|
||||
country: 'in',
|
||||
short: 'BJP',
|
||||
name: 'Bharatiya Janata Party',
|
||||
econ: 6.0,
|
||||
social: 8.5,
|
||||
note: 'Hindu-nationalist, pro-business, big welfare schemes'
|
||||
},
|
||||
|
||||
{ country: 'br', short: 'PT', name: 'Partido dos Trabalhadores', econ: 2.5, social: 4.5 },
|
||||
{ country: 'br', short: 'MDB', name: 'Movimento Democrático Brasileiro', econ: 6.0, social: 6.0 },
|
||||
{ country: 'br', short: 'PL', name: 'Partido Liberal', econ: 7.5, social: 9.0 },
|
||||
|
||||
{ country: 'ca', short: 'NDP', name: 'New Democratic Party', econ: 2.5, social: 2.5 },
|
||||
{ country: 'ca', short: 'LPC', name: 'Liberal Party of Canada', econ: 4.8, social: 3.0 },
|
||||
{ country: 'ca', short: 'CPC', name: 'Conservative Party of Canada', econ: 7.5, social: 7.0 }
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Country profiles, kept short enough to scan
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Country profiles, kept short enough to scan.
|
||||
export type Profile = {
|
||||
id: string;
|
||||
/** What "normal" politics looks like, in one line. */
|
||||
@@ -714,263 +94,43 @@ export type Profile = {
|
||||
blindSpots: string[];
|
||||
};
|
||||
|
||||
export const profiles: Profile[] = [
|
||||
{
|
||||
id: 'us',
|
||||
center: 'Individual liberty and suspicion of government, on both sides',
|
||||
fights: ['abortion', 'guns', 'race', 'immigration', 'religion'],
|
||||
timeline: [
|
||||
{ year: '1776', label: 'revolt against a distant state' },
|
||||
{ year: '1861', label: 'civil war over slavery' },
|
||||
{ year: '1945', label: 'wins WWII, superpower' },
|
||||
{ year: '1964', label: 'Civil Rights Act' },
|
||||
{ year: '1991', label: 'wins the Cold War' }
|
||||
],
|
||||
blindSpots: [
|
||||
'assumes its left/right is universal',
|
||||
'partisan media bubbles',
|
||||
'little view of other systems'
|
||||
]
|
||||
},
|
||||
{
|
||||
id: 'nl',
|
||||
center: 'Coalition consensus ("polder model") around a strong welfare state',
|
||||
fights: ['immigration', 'housing', 'nitrogen & farmers', 'EU'],
|
||||
timeline: [
|
||||
{ year: '1600s', label: 'Golden Age, trade empire' },
|
||||
{ year: '1940', label: 'German occupation' },
|
||||
{ year: '1949', label: 'loses Indonesia' },
|
||||
{ year: '1960s', label: 'pillars break, secularization' },
|
||||
{ year: '2002', label: 'Fortuyn, immigration debate' }
|
||||
],
|
||||
blindSpots: ['tolerance self-image vs discrimination', 'colonial past taught thinly']
|
||||
},
|
||||
{
|
||||
id: 'de',
|
||||
center: 'Social market economy, fiscal caution, "never again"',
|
||||
fights: ['migration', 'energy & climate', 'East vs West', 'debt brake'],
|
||||
timeline: [
|
||||
{ year: '1923', label: 'hyperinflation' },
|
||||
{ year: '1933', label: 'Nazi dictatorship' },
|
||||
{ year: '1949', label: 'split + economic miracle' },
|
||||
{ year: '1990', label: 'reunification' },
|
||||
{ year: '2015', label: 'refugee crisis' }
|
||||
],
|
||||
blindSpots: ['debt as moral failing', 'Russian gas as "just trade"']
|
||||
},
|
||||
{
|
||||
id: 'fr',
|
||||
center: 'Strong central state, laïcité, protest as normal politics',
|
||||
fights: ['pensions', 'Paris vs periphery', 'Islam & laïcité', 'EU'],
|
||||
timeline: [
|
||||
{ year: '1789', label: 'Revolution invents left/right' },
|
||||
{ year: '1940', label: 'defeat and Vichy' },
|
||||
{ year: '1962', label: 'loses Algerian war' },
|
||||
{ year: '1968', label: "May '68" },
|
||||
{ year: '2018', label: 'Yellow Vests' }
|
||||
],
|
||||
blindSpots: ['colorblind universalism hides racism', 'influence in former African colonies']
|
||||
},
|
||||
{
|
||||
id: 'gb',
|
||||
center: 'Pragmatic and market-friendly, but the NHS is sacred',
|
||||
fights: ['Brexit', 'class', 'North vs South', 'Scotland'],
|
||||
timeline: [
|
||||
{ year: '1900', label: 'largest empire in history' },
|
||||
{ year: '1940', label: '"stood alone"' },
|
||||
{ year: '1948', label: 'NHS founded' },
|
||||
{ year: '1979', label: 'Thatcher' },
|
||||
{ year: '2016', label: 'Brexit vote' }
|
||||
],
|
||||
blindSpots: ['empire taught selectively', 'overrates the "special relationship"']
|
||||
},
|
||||
{
|
||||
id: 'se',
|
||||
center: 'High-trust, high-tax welfare state; very secular and individualist',
|
||||
fights: ['immigration', 'gang violence', 'private welfare', 'NATO'],
|
||||
timeline: [
|
||||
{ year: '1814', label: 'last war fought' },
|
||||
{ year: '1932', label: "Social Democrats' era" },
|
||||
{ year: '1939', label: 'neutral in WWII' },
|
||||
{ year: '2015', label: 'record asylum intake' },
|
||||
{ year: '2024', label: 'joins NATO' }
|
||||
],
|
||||
blindSpots: ['consensus makes disagreement taboo', 'neutrality myth (iron to Germany)']
|
||||
},
|
||||
{
|
||||
id: 'pl',
|
||||
center: 'Pro-NATO and wary of Russia across all parties',
|
||||
fights: ['Catholic vs liberal', 'abortion', 'rule of law', 'EU'],
|
||||
timeline: [
|
||||
{ year: '1795', label: 'erased by partition' },
|
||||
{ year: '1939', label: 'invaded by Germany and USSR' },
|
||||
{ year: '1980', label: 'Solidarity movement' },
|
||||
{ year: '1989', label: 'communism falls' },
|
||||
{ year: '2004', label: 'joins the EU' }
|
||||
],
|
||||
blindSpots: ['victimhood crowds out dark chapters', 'distrust of Germany']
|
||||
},
|
||||
{
|
||||
id: 'ru',
|
||||
center: 'Strong state and great-power status; opposition repressed',
|
||||
fights: ['largely suppressed in public'],
|
||||
timeline: [
|
||||
{ year: '1812', label: 'Napoleon invades' },
|
||||
{ year: '1941', label: 'Nazi invasion, ~27M dead' },
|
||||
{ year: '1991', label: 'USSR collapses' },
|
||||
{ year: '1990s', label: 'chaos and poverty' },
|
||||
{ year: '2022', label: 'full invasion of Ukraine' }
|
||||
],
|
||||
blindSpots: [
|
||||
'state-controlled media',
|
||||
'neighbors seen as sphere of influence',
|
||||
'Soviet crimes downplayed'
|
||||
]
|
||||
},
|
||||
{
|
||||
id: 'cn',
|
||||
center: 'Order, growth and national rejuvenation, led by the Party',
|
||||
fights: ['youth jobs', 'property crisis', 'local corruption'],
|
||||
timeline: [
|
||||
{ year: '1839', label: 'Opium Wars, "humiliation"' },
|
||||
{ year: '1937', label: 'Japanese invasion' },
|
||||
{ year: '1966', label: 'Cultural Revolution' },
|
||||
{ year: '1978', label: 'reform and opening' },
|
||||
{ year: '1989', label: 'Tiananmen (unspoken)' }
|
||||
],
|
||||
blindSpots: ['censorship shapes history', 'criticism read as new humiliation']
|
||||
},
|
||||
{
|
||||
id: 'jp',
|
||||
center: 'Stability and harmony; one party governing since 1955',
|
||||
fights: ['pacifist constitution', 'aging & immigration', 'war memory'],
|
||||
timeline: [
|
||||
{ year: '1868', label: 'Meiji modernization' },
|
||||
{ year: '1937', label: 'war in Asia' },
|
||||
{ year: '1945', label: 'atomic bombs, defeat' },
|
||||
{ year: '1947', label: 'pacifist constitution' },
|
||||
{ year: '1990', label: 'bubble bursts' }
|
||||
],
|
||||
blindSpots: ['wartime atrocities taught thinly', 'sees itself as war victim']
|
||||
},
|
||||
{
|
||||
id: 'kr',
|
||||
center: 'Fierce democracy won by protest; hyper-competitive society',
|
||||
fights: ['North Korea', 'Japan', 'gender', 'chaebol power'],
|
||||
timeline: [
|
||||
{ year: '1910', label: 'Japanese colonization' },
|
||||
{ year: '1950', label: 'Korean War' },
|
||||
{ year: '1980', label: 'Gwangju massacre' },
|
||||
{ year: '1987', label: 'democracy' },
|
||||
{ year: '2024', label: 'martial law crisis' }
|
||||
],
|
||||
blindSpots: ['US and Japan seen through party lens']
|
||||
},
|
||||
{
|
||||
id: 'in',
|
||||
center: 'Largest democracy; religion, caste and language drive politics',
|
||||
fights: ['Hindu nationalism', 'caste quotas', 'language', 'center vs states'],
|
||||
timeline: [
|
||||
{ year: '1858', label: 'British Raj' },
|
||||
{ year: '1947', label: 'independence and Partition' },
|
||||
{ year: '1961', label: 'Non-Aligned Movement' },
|
||||
{ year: '1991', label: 'economic liberalization' },
|
||||
{ year: '2014', label: 'BJP majority' }
|
||||
],
|
||||
blindSpots: ['polarized media', 'reduced to poverty & religion by outsiders']
|
||||
},
|
||||
{
|
||||
id: 'br',
|
||||
center: 'Inequality makes redistribution central; religious society',
|
||||
fights: ['Lula vs Bolsonaro', 'crime', 'Amazon vs agribusiness'],
|
||||
timeline: [
|
||||
{ year: '1888', label: 'last to abolish slavery' },
|
||||
{ year: '1964', label: 'military coup' },
|
||||
{ year: '1985', label: 'democracy returns' },
|
||||
{ year: '1994', label: 'ends hyperinflation' },
|
||||
{ year: '2014', label: 'Car Wash scandal' }
|
||||
],
|
||||
blindSpots: ['"racial democracy" myth', 'WhatsApp information bubbles']
|
||||
},
|
||||
{
|
||||
id: 'tr',
|
||||
center: 'National pride, strong state, balancing West and region',
|
||||
fights: ['religious vs secular', 'Kurdish question', 'inflation'],
|
||||
timeline: [
|
||||
{ year: '1915', label: 'Armenian genocide (denied)' },
|
||||
{ year: '1920', label: 'Sèvres, fear of partition' },
|
||||
{ year: '1923', label: "Atatürk's secular republic" },
|
||||
{ year: '1980', label: 'military coup' },
|
||||
{ year: '2016', label: 'failed coup' }
|
||||
],
|
||||
blindSpots: ['genocide denial', 'restricted press', 'Western criticism read as hostility']
|
||||
}
|
||||
];
|
||||
|
||||
export const profileById = new Map(profiles.map((profile) => [profile.id, profile]));
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// One event, many stories
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// One event, many stories.
|
||||
export type Narrative = {
|
||||
event: string;
|
||||
years: string;
|
||||
views: { country: string; story: string }[];
|
||||
};
|
||||
|
||||
export const narratives: Narrative[] = [
|
||||
{
|
||||
event: 'World War II',
|
||||
years: '1939–45',
|
||||
views: [
|
||||
{ country: 'us', story: 'D-Day: democracy beats fascism' },
|
||||
{ country: 'ru', story: 'We broke Hitler, 27M dead' },
|
||||
{ country: 'cn', story: 'Our war began in 1937' },
|
||||
{ country: 'pl', story: 'Liberation became Soviet occupation' },
|
||||
{ country: 'jp', story: 'Hiroshima: we suffered' }
|
||||
]
|
||||
},
|
||||
{
|
||||
event: 'End of the Cold War',
|
||||
years: '1989–91',
|
||||
views: [
|
||||
{ country: 'us', story: 'We won: the end of history' },
|
||||
{ country: 'pl', story: 'We freed ourselves' },
|
||||
{ country: 'ru', story: 'Collapse and humiliation' },
|
||||
{ country: 'cn', story: 'A warning: never lose control' }
|
||||
]
|
||||
},
|
||||
{
|
||||
event: 'Colonialism',
|
||||
years: '1500s–1970s',
|
||||
views: [
|
||||
{ country: 'nl', story: 'Golden Age trade, now debated' },
|
||||
{ country: 'gb', story: 'Empire, remembered fondly by many' },
|
||||
{ country: 'in', story: 'Extraction and Partition' },
|
||||
{ country: 'cn', story: 'Century of humiliation' },
|
||||
{ country: 'br', story: 'Conquest, then US dominance' }
|
||||
]
|
||||
},
|
||||
{
|
||||
event: 'Iraq War',
|
||||
years: '2003',
|
||||
views: [
|
||||
{ country: 'us', story: 'Disarmament, later a mistake' },
|
||||
{ country: 'fr', story: 'We warned you' },
|
||||
{ country: 'iq', story: 'Invasion, then civil war' },
|
||||
{ country: 'ru', story: 'Proof the rules are selective' }
|
||||
]
|
||||
}
|
||||
];
|
||||
export type Source = { name: string; href: string };
|
||||
|
||||
export const sources = [
|
||||
{
|
||||
name: 'World Values Survey / EVS cultural map',
|
||||
href: 'https://www.worldvaluessurvey.org/WVSContents.jsp?CMSID=Findings'
|
||||
},
|
||||
{ name: 'Chapel Hill Expert Survey', href: 'https://www.chesdata.eu/' },
|
||||
{ name: 'Manifesto Project', href: 'https://manifesto-project.wzb.eu/' },
|
||||
{ name: 'Pew Research Center', href: 'https://www.pewresearch.org/global/' },
|
||||
{ name: 'Natural Earth (map shapes)', href: 'https://www.naturalearthdata.com/' }
|
||||
];
|
||||
/** What the page loads: plain arrays in display order. */
|
||||
export type WorldOpinionsData = {
|
||||
countries: Country[];
|
||||
parties: Party[];
|
||||
profiles: Profile[];
|
||||
narratives: Narrative[];
|
||||
sources: Source[];
|
||||
};
|
||||
|
||||
/** The loaded data plus the lookups the components use. */
|
||||
export type WorldOpinions = WorldOpinionsData & {
|
||||
countryById: Map<string, Country>;
|
||||
profileById: Map<string, Profile>;
|
||||
/** Countries with compass parties, in the order their parties are listed. */
|
||||
partyCountries: string[];
|
||||
};
|
||||
|
||||
export function buildWorldOpinions(data: WorldOpinionsData): WorldOpinions {
|
||||
return {
|
||||
...data,
|
||||
countryById: new Map(data.countries.map((country) => [country.id, country])),
|
||||
profileById: new Map(data.profiles.map((profile) => [profile.id, profile])),
|
||||
partyCountries: [...new Set(data.parties.map((party) => party.country))]
|
||||
};
|
||||
}
|
||||
|
||||
const KEY = Symbol('world-opinions');
|
||||
|
||||
/** Called once by the page; the map, panels and compass read it with `worldOpinions()`. */
|
||||
export const provideWorldOpinions = (data: WorldOpinions) => setContext(KEY, data);
|
||||
export const worldOpinions = () => getContext<WorldOpinions>(KEY);
|
||||
|
||||
4
src/routes/projects/world-opinions/+page.server.ts
Normal file
4
src/routes/projects/world-opinions/+page.server.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
import { loadWorldOpinions } from '#lib/server/world-opinions.js';
|
||||
import type { PageServerLoad } from './$types';
|
||||
|
||||
export const load: PageServerLoad = async () => ({ wo: await loadWorldOpinions() });
|
||||
@@ -5,9 +5,14 @@
|
||||
import CultureMap from '#lib/world-opinions/CultureMap.svelte';
|
||||
import PartyCompass from '#lib/world-opinions/PartyCompass.svelte';
|
||||
import WorldMap, { type MapMode } from '#lib/world-opinions/WorldMap.svelte';
|
||||
import { countries, countryById, narratives, sources } from '#lib/world-opinions/data.js';
|
||||
import { buildWorldOpinions, provideWorldOpinions } from '#lib/world-opinions/data.js';
|
||||
import { Button } from '#lib/components/ui/button/index.js';
|
||||
|
||||
let { data } = $props();
|
||||
// Static per load (stats.wo_*), so it is read once and shared with the components.
|
||||
const wo = provideWorldOpinions(buildWorldOpinions(data.wo));
|
||||
const { countries, countryById, narratives, sources } = wo;
|
||||
|
||||
const modes: { id: MapMode; label: string }[] = [
|
||||
{ id: 'liberal', label: 'what "liberal" means' },
|
||||
{ id: 'divide', label: 'what left vs right is about' },
|
||||
|
||||
Reference in New Issue
Block a user