website/scripts/stats/bis.ts

// A5 BIS central bank policy rates (WS_CBPOL), monthly and daily, in one call each; plus the BIS
// long consumer price series (WS_LONG_CPI, index 2010 = 100, monthly) → `bis:cpi:<area>`.
import { getText, log, run, sql, writeSeries, type ObsRow, type SeriesRow } from './common';

const AREAS: Record<string, { name: string; region: string | null }> = {
	US: { name: 'Federal Reserve (US)', region: 'USA' },
	XM: { name: 'ECB (euro area)', region: 'EA' },
	KR: { name: 'Bank of Korea', region: 'KOR' },
	CN: { name: "People's Bank of China", region: 'CHN' },
	GB: { name: 'Bank of England', region: 'GBR' },
	JP: { name: 'Bank of Japan', region: 'JPN' },
	CA: { name: 'Bank of Canada', region: 'CAN' }
};

/** WS_LONG_CPI areas (no Curaçao: its CPI comes from CBS Curaçao). */
const CPI_AREAS: Record<string, string> = {
	CA: 'CAN',
	CN: 'CHN',
	JP: 'JPN',
	KR: 'KOR',
	US: 'USA',
	NL: 'NLD',
	XM: 'EA',
	DE: 'DEU',
	FR: 'FRA',
	BE: 'BEL',
	GB: 'GBR',
	IN: 'IND',
	BR: 'BRA',
	RU: 'RUS'
};

function parseCsv(text: string): Record<string, string>[] {
	const lines = text.trim().split(/\r?\n/);
	const split = (line: string) => {
		const out: string[] = [];
		let cur = '';
		let q = false;
		for (const ch of line) {
			if (ch === '"') q = !q;
			else if (ch === ',' && !q) {
				out.push(cur);
				cur = '';
			} else cur += ch;
		}
		out.push(cur);
		return out;
	};
	const head = split(lines[0]);
	return lines.slice(1).map((l) => Object.fromEntries(split(l).map((v, i) => [head[i], v])));
}

export async function bis(opts: { job: string }) {
	await run('bis', opts.job, async () => {
		let total = 0;
		for (const freq of ['M', 'D'] as const) {
			const csv = await getText(
				`https://stats.bis.org/api/v2/data/dataflow/BIS/WS_CBPOL/1.0/${freq}.${Object.keys(AREAS).join('+')}?format=csv`
			);
			const rows = parseCsv(csv)
				.filter((r) => r.TIME_PERIOD && r.REF_AREA)
				.map((r) => ({
					ref_area: r.REF_AREA,
					freq,
					period: freq === 'M' ? `${r.TIME_PERIOD.slice(0, 7)}-01` : r.TIME_PERIOD.slice(0, 10),
					rate: r.OBS_VALUE === '' || r.OBS_VALUE === 'NaN' ? null : Number(r.OBS_VALUE)
				}));
			await sql.begin(async (tx) => {
				await tx`delete from stats.bis_policy_rate where freq = ${freq}`;
				for (let i = 0; i < rows.length; i += 5000)
					await tx`insert into stats.bis_policy_rate ${tx(rows.slice(i, i + 5000))} on conflict do nothing`;
			});
			const series: SeriesRow[] = Object.entries(AREAS).map(([code, a]) => ({
				series_id: `bis:${freq}:${code}`,
				source_id: 'bis',
				source_code: `WS_CBPOL/${freq}.${code}`,
				title: `${a.name} policy rate`,
				unit: '%',
				freq,
				region_code: a.region,
				theme: 'economy',
				meta: {}
			}));
			const obs: ObsRow[] = rows
				.filter((r) => r.rate != null)
				.map((r) => ({
					series_id: `bis:${freq}:${r.ref_area}`,
					period: r.period,
					value: r.rate,
					status: null
				}));
			await writeSeries(series, obs);
			log(`bis ${freq}: ${rows.length} rows`);
			total += rows.length;
		}
		// Long consumer prices: unit 628 = index, 2010 = 100.
		const cpiCsv = await getText(
			`https://stats.bis.org/api/v2/data/dataflow/BIS/WS_LONG_CPI/1.0/M.${Object.keys(CPI_AREAS).join('+')}.628?format=csv`
		);
		const cpiObs: ObsRow[] = parseCsv(cpiCsv)
			.filter((r) => r.REF_AREA in CPI_AREAS && /^\d{4}-\d{2}/.test(r.TIME_PERIOD ?? ''))
			.filter((r) => r.OBS_VALUE !== '' && r.OBS_VALUE !== 'NaN')
			.map((r) => ({
				series_id: `bis:cpi:${r.REF_AREA}`,
				period: `${r.TIME_PERIOD.slice(0, 7)}-01`,
				value: Number(r.OBS_VALUE),
				status: r.OBS_STATUS && r.OBS_STATUS !== 'A' ? r.OBS_STATUS : null
			}));
		const cpiSeries: SeriesRow[] = Object.entries(CPI_AREAS)
			.filter(([code]) => cpiObs.some((o) => o.series_id === `bis:cpi:${code}`))
			.map(([code, region]) => ({
				series_id: `bis:cpi:${code}`,
				source_id: 'bis',
				source_code: `WS_LONG_CPI/M.${code}.628`,
				title: 'Consumer prices (BIS long series)',
				unit: 'Index 2010=100',
				freq: 'M',
				region_code: region,
				theme: 'economy',
				meta: { url: 'https://data.bis.org/topics/CPI' }
			}));
		await writeSeries(cpiSeries, cpiObs);
		log(`bis cpi: ${cpiObs.length} obs`);
		total += cpiObs.length;
		return total;
	});
}