stats: new public stats schema (source/series/obs/region/fetch_run + raw CBS, Eurostat, FRED, BIS tables) and the stats collector (scripts/stats-fetch.ts); preflight checks the stats tables

scripts/stats/bis.ts

@@ -0,0 +1,78 @@
+// A5 BIS central bank policy rates (WS_CBPOL), monthly and daily, in one call each.
+import { getText, log, run, sql, writeSeries, type ObsRow, type SeriesRow } from './common';
+
+const AREAS: Record<string, { name: string; region: string | null }> = {
+	US: { name: 'Federal Reserve (US)', region: 'USA' },
+	XM: { name: 'ECB (euro area)', region: 'EA' },
+	KR: { name: 'Bank of Korea', region: 'KOR' },
+	CN: { name: "People's Bank of China", region: 'CHN' },
+	GB: { name: 'Bank of England', region: 'GBR' },
+	JP: { name: 'Bank of Japan', region: 'JPN' }
+};
+
+function parseCsv(text: string): Record<string, string>[] {
+	const lines = text.trim().split(/\r?\n/);
+	const split = (line: string) => {
+		const out: string[] = [];
+		let cur = '';
+		let q = false;
+		for (const ch of line) {
+			if (ch === '"') q = !q;
+			else if (ch === ',' && !q) {
+				out.push(cur);
+				cur = '';
+			} else cur += ch;
+		}
+		out.push(cur);
+		return out;
+	};
+	const head = split(lines[0]);
+	return lines.slice(1).map((l) => Object.fromEntries(split(l).map((v, i) => [head[i], v])));
+}
+
+export async function bis(opts: { job: string }) {
+	await run('bis', opts.job, async () => {
+		let total = 0;
+		for (const freq of ['M', 'D'] as const) {
+			const csv = await getText(
+				`https://stats.bis.org/api/v2/data/dataflow/BIS/WS_CBPOL/1.0/${freq}.${Object.keys(AREAS).join('+')}?format=csv`
+			);
+			const rows = parseCsv(csv)
+				.filter((r) => r.TIME_PERIOD && r.REF_AREA)
+				.map((r) => ({
+					ref_area: r.REF_AREA,
+					freq,
+					period: freq === 'M' ? `${r.TIME_PERIOD.slice(0, 7)}-01` : r.TIME_PERIOD.slice(0, 10),
+					rate: r.OBS_VALUE === '' || r.OBS_VALUE === 'NaN' ? null : Number(r.OBS_VALUE)
+				}));
+			await sql.begin(async (tx) => {
+				await tx`delete from stats.bis_policy_rate where freq = ${freq}`;
+				for (let i = 0; i < rows.length; i += 5000)
+					await tx`insert into stats.bis_policy_rate ${tx(rows.slice(i, i + 5000))} on conflict do nothing`;
+			});
+			const series: SeriesRow[] = Object.entries(AREAS).map(([code, a]) => ({
+				series_id: `bis:${freq}:${code}`,
+				source_id: 'bis',
+				source_code: `WS_CBPOL/${freq}.${code}`,
+				title: `${a.name} policy rate`,
+				unit: '%',
+				freq,
+				region_code: a.region,
+				theme: 'economy',
+				meta: {}
+			}));
+			const obs: ObsRow[] = rows
+				.filter((r) => r.rate != null)
+				.map((r) => ({
+					series_id: `bis:${freq}:${r.ref_area}`,
+					period: r.period,
+					value: r.rate,
+					status: null
+				}));
+			await writeSeries(series, obs);
+			log(`bis ${freq}: ${rows.length} rows`);
+			total += rows.length;
+		}
+		return total;
+	});
+}