stan/website
1
0
Fork 0
Code Issues 13 Pull Requests Actions Packages Projects Releases Wiki Activity

stats step 3: elections, party positions and opinion feeds (schema, collectors, timers); world-opinions data moved into stats.wo_*
All checks were successful
Deploy production / deploy (push) Successful in 1m12s
Details

Browse Source
This commit is contained in:
Stan Runge
2026-10-25 22:16:25 +00:00
parent 030f3c08ee
commit 42c9351ebd
37 changed files with 75721 additions and 892 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
scripts/stats/common.ts
View File

@@ -37,7 +37,17 @@ const hostGapMs: Record<string, number> = {
'cuatro.sim-cdn.nl': 1000,
'sdmx.ilo.org': 3000,
'unstats.un.org': 1500,
'comtradeapi.un.org': 2500
'comtradeapi.un.org': 2500,
'data.overheid.nl': 500,
'dataverse.harvard.edu': 1000,
'github.com': 1000,
'partyfacts.herokuapp.com': 2000,
'en.wikipedia.org': 1000,
'data.europa.eu': 1000,
'webgate.ec.europa.eu': 2000,
'files.worldhappiness.report': 1000,
'trends.google.com': 3000,
'api.gdeltproject.org': 5500
};
const lastHit = new Map<string, number>();
@@ -54,7 +64,7 @@ export async function http(url: string, init: RequestInit = {}, tries = 4): Prom
response = await fetch(url, {
...init,
headers: { 'user-agent': USER_AGENT, accept: '*/*', ...(init.headers ?? {}) },
signal: AbortSignal.timeout(180_000)
signal: init.signal ?? AbortSignal.timeout(180_000)
});
if (response.ok) return response;
if (response.status !== 429 && response.status < 500) {
@@ -248,6 +258,74 @@ export const SOURCES = [
licence: 'Free use with attribution (UN Comtrade public preview)',
key_required: false,
attribution: 'Source: UN Comtrade Database (United Nations Statistics Division)'
},
{
source_id: 'kiesraad',
name: 'Kiesraad election results (data.overheid.nl)',
base_url: 'https://data.overheid.nl/dataset?q=verkiezingsuitslag',
licence: 'CC0 1.0',
key_required: false,
attribution: 'Source: Kiesraad, verkiezingsuitslagen (data.overheid.nl), CC0'
},
{
source_id: 'mitelection',
name: 'MIT Election Data and Science Lab, U.S. President 1976–2024',
base_url: 'https://doi.org/10.7910/DVN/42MVDX',
licence: 'CC0 1.0',
key_required: false,
attribution:
'Source: MIT Election Data and Science Lab, U.S. President 1976–2024 (Harvard Dataverse), CC0'
},
{
source_id: 'ches',
name: 'Chapel Hill Expert Survey (trend file) + V-Party + Party Facts',
base_url: 'https://www.chesdata.eu',
licence:
'Free for non-commercial research use with citation (CHES); V-Dem CC BY-SA 4.0; Party Facts CC BY-SA 4.0',
key_required: false,
attribution:
'Sources: Chapel Hill Expert Survey 1999–2024 (Jolly et al.); V-Dem V-Party v2 (Lührmann et al.); Party Facts (Döring & Regel)'
},
{
source_id: 'wikipedia',
name: 'Wikipedia opinion-polling tables',
base_url: 'https://en.wikipedia.org',
licence: 'CC BY-SA 4.0',
key_required: false,
attribution:
'Source: Wikipedia opinion-polling articles and their cited pollsters, CC BY-SA 4.0'
},
{
source_id: 'eurobarometer',
name: 'Standard Eurobarometer (volume A tables)',
base_url: 'https://data.europa.eu',
licence: 'Reuse authorised with acknowledgement (Commission Decision 2011/833/EU)',
key_required: false,
attribution: 'Source: European Commission, Standard Eurobarometer, © European Union'
},
{
source_id: 'whr',
name: 'World Happiness Report (Figure 2.1 data)',
base_url: 'https://worldhappiness.report',
licence: 'Free reuse with attribution',
key_required: false,
attribution: 'Source: World Happiness Report (Wellbeing Research Centre, Gallup World Poll)'
},
{
source_id: 'gtrends',
name: 'Google Trends, trending now (RSS)',
base_url: 'https://trends.google.com/trending/rss',
licence: 'Google Terms of Service; shown with attribution',
key_required: false,
attribution: 'Source: Google Trends'
},
{
source_id: 'gdelt',
name: 'GDELT Project DOC 2.0 API',
base_url: 'https://api.gdeltproject.org/api/v2/doc/doc',
licence: 'Open, unrestricted use with citation',
key_required: false,
attribution: 'Source: The GDELT Project'
}
] as const;