stats: core-7 economy page with CPI index levels, rebase and categories

- /stats/economy defaults multi-country charts to CAN KOR CHN JPN USA NLD CUW,
  with add-a-country chips and an EU 27 comparison line
- CPI index levels rebased to a chosen base year, index/YoY toggle,
  categories (food, energy, housing, core) where the source has them
- new no-key collectors: oecd (CPI categories, 10y yields), statcan (CPI);
  bis WS_LONG_CPI, FRED US CPI categories, Eurostat prc_hicp_minr, WB FP.CPI.TOTL
- Canada added as a region; CBS Curacao fallbacks for growth/unemployment
- timers stats-oecd and stats-statcan

scripts/stats/bis.ts

@@ -1,4 +1,5 @@
-// A5 BIS central bank policy rates (WS_CBPOL), monthly and daily, in one call each.
+// A5 BIS central bank policy rates (WS_CBPOL), monthly and daily, in one call each; plus the BIS
+// long consumer price series (WS_LONG_CPI, index 2010 = 100, monthly) → `bis:cpi:<area>`.
 import { getText, log, run, sql, writeSeries, type ObsRow, type SeriesRow } from './common';
 
 const AREAS: Record<string, { name: string; region: string | null }> = {
@@ -7,7 +8,26 @@ const AREAS: Record<string, { name: string; region: string | null }> = {
 	KR: { name: 'Bank of Korea', region: 'KOR' },
 	CN: { name: "People's Bank of China", region: 'CHN' },
 	GB: { name: 'Bank of England', region: 'GBR' },
-	JP: { name: 'Bank of Japan', region: 'JPN' }
+	JP: { name: 'Bank of Japan', region: 'JPN' },
+	CA: { name: 'Bank of Canada', region: 'CAN' }
+};
+
+/** WS_LONG_CPI areas (no Curaçao: its CPI comes from CBS Curaçao). */
+const CPI_AREAS: Record<string, string> = {
+	CA: 'CAN',
+	CN: 'CHN',
+	JP: 'JPN',
+	KR: 'KOR',
+	US: 'USA',
+	NL: 'NLD',
+	XM: 'EA',
+	DE: 'DEU',
+	FR: 'FRA',
+	BE: 'BEL',
+	GB: 'GBR',
+	IN: 'IND',
+	BR: 'BRA',
+	RU: 'RUS'
 };
 
 function parseCsv(text: string): Record<string, string>[] {
@@ -73,6 +93,35 @@ export async function bis(opts: { job: string }) {
 			log(`bis ${freq}: ${rows.length} rows`);
 			total += rows.length;
 		}
+		// Long consumer prices: unit 628 = index, 2010 = 100.
+		const cpiCsv = await getText(
+			`https://stats.bis.org/api/v2/data/dataflow/BIS/WS_LONG_CPI/1.0/M.${Object.keys(CPI_AREAS).join('+')}.628?format=csv`
+		);
+		const cpiObs: ObsRow[] = parseCsv(cpiCsv)
+			.filter((r) => r.REF_AREA in CPI_AREAS && /^\d{4}-\d{2}/.test(r.TIME_PERIOD ?? ''))
+			.filter((r) => r.OBS_VALUE !== '' && r.OBS_VALUE !== 'NaN')
+			.map((r) => ({
+				series_id: `bis:cpi:${r.REF_AREA}`,
+				period: `${r.TIME_PERIOD.slice(0, 7)}-01`,
+				value: Number(r.OBS_VALUE),
+				status: r.OBS_STATUS && r.OBS_STATUS !== 'A' ? r.OBS_STATUS : null
+			}));
+		const cpiSeries: SeriesRow[] = Object.entries(CPI_AREAS)
+			.filter(([code]) => cpiObs.some((o) => o.series_id === `bis:cpi:${code}`))
+			.map(([code, region]) => ({
+				series_id: `bis:cpi:${code}`,
+				source_id: 'bis',
+				source_code: `WS_LONG_CPI/M.${code}.628`,
+				title: 'Consumer prices (BIS long series)',
+				unit: 'Index 2010=100',
+				freq: 'M',
+				region_code: region,
+				theme: 'economy',
+				meta: { url: 'https://data.bis.org/topics/CPI' }
+			}));
+		await writeSeries(cpiSeries, cpiObs);
+		log(`bis cpi: ${cpiObs.length} obs`);
+		total += cpiObs.length;
 		return total;
 	});
 }

scripts/stats/common.ts

@@ -24,6 +24,7 @@ const hostGapMs: Record<string, number> = {
 	'fred.stlouisfed.org': 1000,
 	'ec.europa.eu': 500,
 	'stats.bis.org': 1000,
+	'www150.statcan.gc.ca': 1000,
 	'api.worldbank.org': 1000,
 	'www.imf.org': 1500,
 	'ourworldindata.org': 2000,
@@ -150,11 +151,28 @@ export const SOURCES = [
 	},
 	{
 		source_id: 'bis',
-		name: 'BIS central bank policy rates',
-		base_url: 'https://stats.bis.org/api/v2/data/dataflow/BIS/WS_CBPOL/1.0',
+		name: 'BIS (central bank policy rates, long consumer price series)',
+		base_url: 'https://stats.bis.org/api/v2/data/dataflow/BIS',
 		licence: 'Free reuse with attribution',
 		key_required: false,
-		attribution: 'Source: BIS, central bank policy rates (WS_CBPOL)'
+		attribution:
+			'Source: BIS, central bank policy rates (WS_CBPOL) and consumer prices (WS_LONG_CPI)'
+	},
+	{
+		source_id: 'oecd',
+		name: 'OECD Data Explorer (SDMX)',
+		base_url: 'https://sdmx.oecd.org/public/rest/data',
+		licence: 'CC BY 4.0',
+		key_required: false,
+		attribution: 'Source: OECD Data Explorer (prices; financial market statistics), CC BY 4.0'
+	},
+	{
+		source_id: 'statcan',
+		name: 'Statistics Canada (Web Data Service)',
+		base_url: 'https://www150.statcan.gc.ca/t1/wds/rest',
+		licence: 'Statistics Canada Open Licence',
+		key_required: false,
+		attribution: 'Source: Statistics Canada, table 18-10-0004-01 (Statistics Canada Open Licence)'
 	},
 	{
 		source_id: 'worldbank',
@@ -395,6 +413,7 @@ export const REGIONS: { region_code: string; name: string; kind: string; parent:
 		{ region_code: 'AUT', name: 'Austria', kind: 'country', parent: 'EU27' },
 		{ region_code: 'GBR', name: 'United Kingdom', kind: 'country', parent: 'WLD' },
 		{ region_code: 'JPN', name: 'Japan', kind: 'country', parent: 'WLD' },
+		{ region_code: 'CAN', name: 'Canada', kind: 'country', parent: 'WLD' },
 		{ region_code: 'IND', name: 'India', kind: 'country', parent: 'WLD' },
 		{ region_code: 'BRA', name: 'Brazil', kind: 'country', parent: 'WLD' },
 		{ region_code: 'RUS', name: 'Russia', kind: 'country', parent: 'WLD' },
@@ -415,6 +434,7 @@ export const COMPARE_ISO3 = [
 	'KOR',
 	'CHN',
 	'CUW',
+	'CAN',
 	'DEU',
 	'FRA',
 	'BEL',

scripts/stats/eurostat.ts

@@ -87,7 +87,15 @@ export const EUROSTAT_DATASETS: Dataset[] = [
 		params: {},
 		geos: ['NL', 'DE', 'FR', 'IT', 'BE', 'ES', 'AT', 'EA']
 	},
-	{ code: 'prc_hicp_minr', theme: 'economy', params: { coicop18: 'TOTAL', unit: 'RCH_A' } },
+	{
+		// Rates for every geo; index levels (2015 = 100) and the main categories feed the CPI charts.
+		code: 'prc_hicp_minr',
+		theme: 'economy',
+		params: {
+			coicop18: ['TOTAL', 'CP01', 'CP04', 'NRG', 'TOT_X_NRG_FOOD'],
+			unit: ['RCH_A', 'I15']
+		}
+	},
 	{ code: 'prc_hicp_manr', theme: 'economy', params: { coicop: 'CP00', unit: 'RCH_A' } },
 	{ code: 'prc_hpi_q', theme: 'economy', params: { purchase: 'TOTAL', unit: ['I15_Q', 'RCH_A'] } },
 	{ code: 'road_eqr_carpda', theme: 'energy', params: { unit: 'NR' } },

scripts/stats/fred.ts

@@ -155,6 +155,21 @@ export const FRED_SERIES: Fred[] = [
 		region: 'USA',
 		theme: 'economy'
 	},
+	...(
+		[
+			['CPIUFDSL', 'food'],
+			['CPIHOSSL', 'housing'],
+			['CPIENGSL', 'energy'],
+			['CPILFESL', 'all items less food and energy']
+		] as const
+	).map(([id, what]) => ({
+		id,
+		title: `US consumer price index (CPI-U), ${what}`,
+		unit: 'Index 1982-84=100',
+		freq: 'M' as const,
+		region: 'USA',
+		theme: 'economy'
+	})),
 	{
 		id: 'UNRATE',
 		title: 'US unemployment rate',

scripts/stats/oecd.ts

@@ -0,0 +1,129 @@
+// OECD Data Explorer SDMX (no key, ≤20 requests a minute): consumer price indices by category
+// (DF_PRICES_ALL, 2015 = 100) for the countries without a national feed here, and long-term
+// (10-year) government bond yields (DF_FINMARK). Normalised only: `oecd:cpi:<ISO3>:<category>`
+// and `oecd:irlt:<ISO3>`.
+import { http, log, run, writeSeries, type ObsRow, type SeriesRow } from './common';
+
+const BASE = 'https://sdmx.oecd.org/public/rest/data';
+const CSV = { accept: 'application/vnd.sdmx.data+csv; charset=utf-8' };
+
+/** CPI categories (OECD COICOP codes) → our short names. */
+export const OECD_CPI_CATEGORIES: Record<string, { name: string; title: string }> = {
+	_T: { name: 'total', title: 'Consumer price index, all items' },
+	CP01: { name: 'food', title: 'Consumer price index, food and non-alcoholic beverages' },
+	CP04: {
+		name: 'housing',
+		title: 'Consumer price index, housing, water, electricity, gas and other fuels'
+	},
+	CP045_0722: { name: 'energy', title: 'Consumer price index, energy' },
+	_TXCP01_NRG: { name: 'core', title: 'Consumer price index, all items excluding food and energy' }
+};
+/** National CPI comes from CBS, BLS (FRED), StatCan and CBS Curaçao; OECD fills KOR, JPN, CHN. */
+const CPI_AREAS = ['KOR', 'JPN', 'CHN'];
+const IRLT_AREAS = [
+	'CAN',
+	'JPN',
+	'KOR',
+	'CHN',
+	'USA',
+	'NLD',
+	'DEU',
+	'FRA',
+	'GBR',
+	'BEL',
+	'IND',
+	'BRA',
+	'RUS',
+	'EA20'
+];
+const region = (a: string) => (a === 'EA20' ? 'EA' : a);
+
+function parseCsv(text: string): Record<string, string>[] {
+	const lines = text.trim().split(/\r?\n/);
+	const head = lines[0].split(',');
+	return lines.slice(1).map((l) => Object.fromEntries(l.split(',').map((v, i) => [head[i], v])));
+}
+
+async function pull(
+	path: string,
+	key: (r: Record<string, string>) => { id: string; region: string; code: string } | null,
+	meta: (code: string) => { title: string; unit: string }
+) {
+	const rows = parseCsv(await (await http(`${BASE}/${path}`, { headers: CSV })).text());
+	const series = new Map<string, SeriesRow>();
+	const obs: ObsRow[] = [];
+	for (const r of rows) {
+		const k = key(r);
+		if (!k || !/^\d{4}-\d{2}$/.test(r.TIME_PERIOD ?? '') || r.OBS_VALUE === '') continue;
+		const value = Number(r.OBS_VALUE);
+		if (!Number.isFinite(value)) continue;
+		if (!series.has(k.id)) {
+			const m = meta(k.code);
+			series.set(k.id, {
+				series_id: k.id,
+				source_id: 'oecd',
+				source_code: `${path.split('/')[0]}/${k.code}`,
+				title: m.title,
+				unit: m.unit,
+				freq: 'M',
+				region_code: k.region,
+				theme: 'economy',
+				meta: { url: 'https://data-explorer.oecd.org/' }
+			});
+		}
+		obs.push({
+			series_id: k.id,
+			period: `${r.TIME_PERIOD}-01`,
+			value,
+			status: r.OBS_STATUS || null
+		});
+	}
+	if (!obs.length) throw new Error(`no rows for ${path.split('?')[0]}`);
+	await writeSeries([...series.values()], obs);
+	return obs.length;
+}
+
+export async function oecd(opts: { only?: string; job: string }) {
+	await run('oecd', opts.job, async (note) => {
+		let total = 0;
+		const want = (j: string) => !opts.only || opts.only.split(',').includes(j);
+		if (want('cpi'))
+			try {
+				const n = await pull(
+					`OECD.SDD.TPS,DSD_PRICES@DF_PRICES_ALL,/${CPI_AREAS.join('+')}.M.N.CPI.IX.${Object.keys(OECD_CPI_CATEGORIES).join('+')}.N._Z?startPeriod=1950-01`,
+					(r) => {
+						const c = OECD_CPI_CATEGORIES[r.EXPENDITURE];
+						return c && r.UNIT_MEASURE === 'IX'
+							? { id: `oecd:cpi:${r.REF_AREA}:${c.name}`, region: r.REF_AREA, code: r.EXPENDITURE }
+							: null;
+					},
+					(code) => ({ title: OECD_CPI_CATEGORIES[code].title, unit: 'Index 2015=100' })
+				);
+				log(`oecd cpi: ${n} obs`);
+				total += n;
+			} catch (error) {
+				note(`cpi: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		if (want('irlt'))
+			try {
+				const n = await pull(
+					`OECD.SDD.STES,DSD_STES@DF_FINMARK,/${IRLT_AREAS.join('+')}.M.IRLT......?startPeriod=1950-01`,
+					(r) =>
+						r.MEASURE === 'IRLT'
+							? {
+									id: `oecd:irlt:${region(r.REF_AREA)}`,
+									region: region(r.REF_AREA),
+									code: r.REF_AREA
+								}
+							: null,
+					() => ({ title: 'Long-term interest rate (10-year government bonds)', unit: '%' })
+				);
+				log(`oecd irlt: ${n} obs`);
+				total += n;
+			} catch (error) {
+				note(`irlt: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		if (!total) throw new Error('nothing loaded');
+		return total;
+	});
+}

scripts/stats/statcan.ts

@@ -0,0 +1,57 @@
+// Statistics Canada Web Data Service (no key): CPI table 18-10-0004-01 (2002 = 100, monthly,
+// not seasonally adjusted) by vector. Normalised only: `statcan:cpi:<category>`.
+import { getJson, log, run, writeSeries, type ObsRow, type SeriesRow } from './common';
+
+export const STATCAN_CPI: { vector: number; name: string; title: string }[] = [
+	{ vector: 41690973, name: 'total', title: 'Canada consumer price index, all items' },
+	{ vector: 41690974, name: 'food', title: 'Canada consumer price index, food' },
+	{ vector: 41691050, name: 'housing', title: 'Canada consumer price index, shelter' },
+	{ vector: 41691239, name: 'energy', title: 'Canada consumer price index, energy' },
+	{
+		vector: 41691233,
+		name: 'core',
+		title: 'Canada consumer price index, all items excluding food and energy'
+	}
+];
+
+type Wds = {
+	status: string;
+	object?: { vectorId: number; vectorDataPoint: { refPer: string; value: number | null }[] };
+}[];
+
+export async function statcan(opts: { job: string }) {
+	await run('statcan', opts.job, async (note) => {
+		const ids = STATCAN_CPI.map((s) => `%22${s.vector}%22`).join(',');
+		const body = await getJson<Wds>(
+			`https://www150.statcan.gc.ca/t1/wds/rest/getDataFromVectorByReferencePeriodRange?vectorIds=${ids}&startRefPeriod=1900-01-01&endReferencePeriod=2100-12-01`
+		);
+		const series: SeriesRow[] = [];
+		const obs: ObsRow[] = [];
+		for (const s of STATCAN_CPI) {
+			const points = body.find((b) => b.object?.vectorId === s.vector)?.object?.vectorDataPoint;
+			if (!points?.length) {
+				note(`v${s.vector}: no data`);
+				continue;
+			}
+			const id = `statcan:cpi:${s.name}`;
+			series.push({
+				series_id: id,
+				source_id: 'statcan',
+				source_code: `18-10-0004-01/v${s.vector}`,
+				title: s.title,
+				unit: 'Index 2002=100',
+				freq: 'M',
+				region_code: 'CAN',
+				theme: 'economy',
+				meta: { url: 'https://www150.statcan.gc.ca/t1/tbl1/en/tv.action?pid=1810000401' }
+			});
+			for (const p of points)
+				if (p.value != null && /^\d{4}-\d{2}-01$/.test(p.refPer))
+					obs.push({ series_id: id, period: p.refPer, value: p.value, status: null });
+		}
+		if (!obs.length) throw new Error('no StatCan observations');
+		await writeSeries(series, obs);
+		log(`statcan: ${obs.length} obs in ${series.length} series`);
+		return obs.length;
+	});
+}

scripts/stats/worldbank.ts

@@ -27,6 +27,7 @@ export const WB_INDICATORS: { id: string; title: string; unit: string; theme: st
 	{ id: 'NY.GDP.MKTP.CD', title: 'GDP', unit: 'current USD', theme: 'countries' },
 	{ id: 'NY.GDP.MKTP.KD.ZG', title: 'GDP growth', unit: '%', theme: 'countries' },
 	{ id: 'FP.CPI.TOTL.ZG', title: 'Inflation, consumer prices', unit: '%', theme: 'countries' },
+	{ id: 'FP.CPI.TOTL', title: 'Consumer price index', unit: 'Index 2010=100', theme: 'economy' },
 	{ id: 'SP.POP.TOTL', title: 'Population', unit: 'people', theme: 'countries' },
 	{ id: 'SM.POP.NETM', title: 'Net migration', unit: 'people', theme: 'countries' },
 	{