website/scripts/stats/oecd.ts

// OECD Data Explorer SDMX (no key, ≤20 requests a minute): consumer price indices by category
// (DF_PRICES_ALL, 2015 = 100) for the countries without a national feed here, and long-term
// (10-year) government bond yields (DF_FINMARK). Normalised only: `oecd:cpi:<ISO3>:<category>`
// and `oecd:irlt:<ISO3>`.
import { http, log, run, writeSeries, type ObsRow, type SeriesRow } from './common';

const BASE = 'https://sdmx.oecd.org/public/rest/data';
const CSV = { accept: 'application/vnd.sdmx.data+csv; charset=utf-8' };

/** CPI categories (OECD COICOP codes) → our short names. */
export const OECD_CPI_CATEGORIES: Record<string, { name: string; title: string }> = {
	_T: { name: 'total', title: 'Consumer price index, all items' },
	CP01: { name: 'food', title: 'Consumer price index, food and non-alcoholic beverages' },
	CP04: {
		name: 'housing',
		title: 'Consumer price index, housing, water, electricity, gas and other fuels'
	},
	CP045_0722: { name: 'energy', title: 'Consumer price index, energy' },
	_TXCP01_NRG: { name: 'core', title: 'Consumer price index, all items excluding food and energy' }
};
/** National CPI comes from CBS, BLS (FRED), StatCan and CBS Curaçao; OECD fills KOR, JPN, CHN. */
const CPI_AREAS = ['KOR', 'JPN', 'CHN'];
const IRLT_AREAS = [
	'CAN',
	'JPN',
	'KOR',
	'CHN',
	'USA',
	'NLD',
	'DEU',
	'FRA',
	'GBR',
	'BEL',
	'IND',
	'BRA',
	'RUS',
	'EA20'
];
const region = (a: string) => (a === 'EA20' ? 'EA' : a);

function parseCsv(text: string): Record<string, string>[] {
	const lines = text.trim().split(/\r?\n/);
	const head = lines[0].split(',');
	return lines.slice(1).map((l) => Object.fromEntries(l.split(',').map((v, i) => [head[i], v])));
}

async function pull(
	path: string,
	key: (r: Record<string, string>) => { id: string; region: string; code: string } | null,
	meta: (code: string) => { title: string; unit: string }
) {
	const rows = parseCsv(await (await http(`${BASE}/${path}`, { headers: CSV })).text());
	const series = new Map<string, SeriesRow>();
	const obs: ObsRow[] = [];
	for (const r of rows) {
		const k = key(r);
		if (!k || !/^\d{4}-\d{2}$/.test(r.TIME_PERIOD ?? '') || r.OBS_VALUE === '') continue;
		const value = Number(r.OBS_VALUE);
		if (!Number.isFinite(value)) continue;
		if (!series.has(k.id)) {
			const m = meta(k.code);
			series.set(k.id, {
				series_id: k.id,
				source_id: 'oecd',
				source_code: `${path.split('/')[0]}/${k.code}`,
				title: m.title,
				unit: m.unit,
				freq: 'M',
				region_code: k.region,
				theme: 'economy',
				meta: { url: 'https://data-explorer.oecd.org/' }
			});
		}
		obs.push({
			series_id: k.id,
			period: `${r.TIME_PERIOD}-01`,
			value,
			status: r.OBS_STATUS || null
		});
	}
	if (!obs.length) throw new Error(`no rows for ${path.split('?')[0]}`);
	await writeSeries([...series.values()], obs);
	return obs.length;
}

export async function oecd(opts: { only?: string; job: string }) {
	await run('oecd', opts.job, async (note) => {
		let total = 0;
		const want = (j: string) => !opts.only || opts.only.split(',').includes(j);
		if (want('cpi'))
			try {
				const n = await pull(
					`OECD.SDD.TPS,DSD_PRICES@DF_PRICES_ALL,/${CPI_AREAS.join('+')}.M.N.CPI.IX.${Object.keys(OECD_CPI_CATEGORIES).join('+')}.N._Z?startPeriod=1950-01`,
					(r) => {
						const c = OECD_CPI_CATEGORIES[r.EXPENDITURE];
						return c && r.UNIT_MEASURE === 'IX'
							? { id: `oecd:cpi:${r.REF_AREA}:${c.name}`, region: r.REF_AREA, code: r.EXPENDITURE }
							: null;
					},
					(code) => ({ title: OECD_CPI_CATEGORIES[code].title, unit: 'Index 2015=100' })
				);
				log(`oecd cpi: ${n} obs`);
				total += n;
			} catch (error) {
				note(`cpi: ${error instanceof Error ? error.message : String(error)}`);
			}
		if (want('irlt'))
			try {
				const n = await pull(
					`OECD.SDD.STES,DSD_STES@DF_FINMARK,/${IRLT_AREAS.join('+')}.M.IRLT......?startPeriod=1950-01`,
					(r) =>
						r.MEASURE === 'IRLT'
							? {
									id: `oecd:irlt:${region(r.REF_AREA)}`,
									region: region(r.REF_AREA),
									code: r.REF_AREA
								}
							: null,
					() => ({ title: 'Long-term interest rate (10-year government bonds)', unit: '%' })
				);
				log(`oecd irlt: ${n} obs`);
				total += n;
			} catch (error) {
				note(`irlt: ${error instanceof Error ? error.message : String(error)}`);
			}
		if (!total) throw new Error('nothing loaded');
		return total;
	});
}