stats step 2: World Bank, IMF WEO, OWID, Energy-Charts, Rijksfinanciën, US Treasury, USAspending collectors + tables, timers

scripts/stats/common.ts

@@ -23,7 +23,15 @@ const hostGapMs: Record<string, number> = {
 	'sdmx.oecd.org': 3100,
 	'fred.stlouisfed.org': 1000,
 	'ec.europa.eu': 500,
-	'stats.bis.org': 1000
+	'stats.bis.org': 1000,
+	'api.worldbank.org': 1000,
+	'www.imf.org': 1500,
+	'ourworldindata.org': 2000,
+	'owid-public.owid.io': 2000,
+	'api.energy-charts.info': 2000,
+	'www.rijksfinancien.nl': 1000,
+	'api.fiscaldata.treasury.gov': 1000,
+	'api.usaspending.gov': 500
 };
 const lastHit = new Map<string, number>();
 
@@ -127,6 +135,64 @@ export const SOURCES = [
 		licence: 'Free reuse with attribution',
 		key_required: false,
 		attribution: 'Source: BIS, central bank policy rates (WS_CBPOL)'
+	},
+	{
+		source_id: 'worldbank',
+		name: 'World Bank World Development Indicators',
+		base_url: 'https://api.worldbank.org/v2',
+		licence: 'CC BY 4.0',
+		key_required: false,
+		attribution: 'Source: World Bank, World Development Indicators, CC BY 4.0'
+	},
+	{
+		source_id: 'imf',
+		name: 'IMF World Economic Outlook (DataMapper)',
+		base_url: 'https://www.imf.org/external/datamapper/api/v1',
+		licence: 'IMF copyright, free reuse with attribution',
+		key_required: false,
+		attribution: 'Source: IMF, World Economic Outlook (DataMapper)'
+	},
+	{
+		source_id: 'owid',
+		name: 'Our World in Data',
+		base_url: 'https://ourworldindata.org',
+		licence: 'CC BY 4.0',
+		key_required: false,
+		attribution:
+			'Source: Our World in Data (Maddison Project Database 2023; Energy Institute, Ember), CC BY 4.0'
+	},
+	{
+		source_id: 'energycharts',
+		name: 'Energy-Charts (Fraunhofer ISE)',
+		base_url: 'https://api.energy-charts.info',
+		licence: 'CC BY 4.0',
+		key_required: false,
+		attribution: 'Source: Energy-Charts.info (Fraunhofer ISE; ENTSO-E, SMARD), CC BY 4.0'
+	},
+	{
+		source_id: 'rijksfinancien',
+		name: 'Rijksfinanciën (Ministerie van Financiën)',
+		base_url: 'https://www.rijksfinancien.nl/open-data/api',
+		licence: 'CC0',
+		key_required: false,
+		attribution: 'Source: Rijksfinanciën open data, Ministerie van Financiën (CC0)'
+	},
+	{
+		source_id: 'ustreasury',
+		name: 'US Treasury Fiscal Data',
+		base_url: 'https://api.fiscaldata.treasury.gov/services/api/fiscal_service',
+		licence: 'Public domain (US government work)',
+		key_required: false,
+		attribution:
+			'Source: U.S. Treasury, Fiscal Data (Monthly Treasury Statement, Debt to the Penny)'
+	},
+	{
+		source_id: 'usaspending',
+		name: 'USAspending.gov',
+		base_url: 'https://api.usaspending.gov/api/v2',
+		licence: 'Public domain (US government work)',
+		key_required: false,
+		attribution: 'Source: USAspending.gov (U.S. Department of the Treasury)'
 	}
 ] as const;
 
@@ -152,6 +218,8 @@ export const REGIONS: { region_code: string; name: string; kind: string; parent:
 		{ region_code: 'GBR', name: 'United Kingdom', kind: 'country', parent: 'WLD' },
 		{ region_code: 'JPN', name: 'Japan', kind: 'country', parent: 'WLD' },
 		{ region_code: 'IND', name: 'India', kind: 'country', parent: 'WLD' },
+		{ region_code: 'BRA', name: 'Brazil', kind: 'country', parent: 'WLD' },
+		{ region_code: 'RUS', name: 'Russia', kind: 'country', parent: 'WLD' },
 		{ region_code: 'CR28', name: 'Oost-Zuid-Holland (COROP)', kind: 'corop', parent: 'NLD' },
 		{ region_code: 'GM0513', name: 'Gouda', kind: 'municipality', parent: 'CR28' },
 		{ region_code: 'GM0627', name: 'Waddinxveen', kind: 'municipality', parent: 'CR28' },
@@ -162,6 +230,46 @@ export const REGIONS: { region_code: string; name: string; kind: string; parent:
 		{ region_code: 'GM0632', name: 'Woerden', kind: 'municipality', parent: 'NLD' }
 	];
 
+/** Countries compared on /stats/countries (ISO3), in display order. */
+export const COMPARE_ISO3 = [
+	'NLD',
+	'USA',
+	'KOR',
+	'CHN',
+	'CUW',
+	'DEU',
+	'FRA',
+	'BEL',
+	'GBR',
+	'JPN',
+	'IND',
+	'BRA',
+	'RUS'
+];
+
+/**
+ * Batch insert inside an open transaction: one jsonb parameter per chunk, so big tables load fast
+ * without hitting the bind-parameter limit. Row keys are column names (snake_case).
+ */
+export async function insertChunks(
+	tx: postgres.TransactionSql,
+	table: string,
+	rows: Record<string, unknown>[],
+	conflict = '',
+	size = 20000
+) {
+	if (!rows.length) return;
+	const cols = Object.keys(rows[0])
+		.map((k) => `"${k}"`)
+		.join(', ');
+	for (let i = 0; i < rows.length; i += size) {
+		await tx.unsafe(
+			`insert into stats.${table} (${cols}) select ${cols} from jsonb_populate_recordset(null::stats.${table}, $1::jsonb) ${conflict}`,
+			[JSON.stringify(rows.slice(i, i + size))] as never
+		);
+	}
+}
+
 /** Gouda and its house-hunting ring (CBS gemeente codes). */
 export const GOUDA_RING = ['GM0513', 'GM0627', 'GM1901', 'GM1892', 'GM1931', 'GM0484', 'GM0632'];