website/scripts/stats/fred.ts

// A7 FRED graph CSV (no key). One call per series → stats.fred_obs and series `fred:<id>`.
import { getText, log, run, sql, writeSeries, type ObsRow } from './common';

type Fred = {
	id: string;
	title: string;
	unit: string;
	freq: 'D' | 'M' | 'Q' | 'A';
	region: string;
	theme: string;
};
export const FRED_SERIES: Fred[] = [
	{
		id: 'DCOILBRENTEU',
		title: 'Brent crude oil spot price',
		unit: 'USD per barrel',
		freq: 'D',
		region: 'WLD',
		theme: 'energy'
	},
	{
		id: 'PNGASEUUSDM',
		title: 'EU natural gas price (TTF, IMF)',
		unit: 'USD per MMBtu',
		freq: 'M',
		region: 'EU27',
		theme: 'energy'
	},
	{
		id: 'DFEDTARU',
		title: 'Fed funds target range, upper limit',
		unit: '%',
		freq: 'D',
		region: 'USA',
		theme: 'economy'
	},
	{
		id: 'DFEDTARL',
		title: 'Fed funds target range, lower limit',
		unit: '%',
		freq: 'D',
		region: 'USA',
		theme: 'economy'
	},
	{
		id: 'DFF',
		title: 'Effective federal funds rate',
		unit: '%',
		freq: 'D',
		region: 'USA',
		theme: 'economy'
	},
	{
		id: 'DGS10',
		title: 'US Treasury 10-year yield',
		unit: '%',
		freq: 'D',
		region: 'USA',
		theme: 'economy'
	},
	{
		id: 'DGS2',
		title: 'US Treasury 2-year yield',
		unit: '%',
		freq: 'D',
		region: 'USA',
		theme: 'economy'
	},
	{
		id: 'IRLTLT01KRM156N',
		title: 'South Korea 10-year government bond yield',
		unit: '%',
		freq: 'M',
		region: 'KOR',
		theme: 'economy'
	},
	{
		id: 'IRLTLT01FRM156N',
		title: 'France 10-year government bond yield',
		unit: '%',
		freq: 'M',
		region: 'FRA',
		theme: 'economy'
	},
	{
		id: 'IRLTLT01DEM156N',
		title: 'Germany 10-year government bond yield',
		unit: '%',
		freq: 'M',
		region: 'DEU',
		theme: 'economy'
	},
	{
		id: 'APU000072610',
		title: 'US average residential electricity price',
		unit: 'USD per kWh',
		freq: 'M',
		region: 'USA',
		theme: 'energy'
	},
	{
		id: 'APU000072620',
		title: 'US average utility (piped) gas price',
		unit: 'USD per therm',
		freq: 'M',
		region: 'USA',
		theme: 'energy'
	},
	{
		id: 'SPASTT01NLM661N',
		title: 'Netherlands share prices (OECD index)',
		unit: 'Index 2015=100',
		freq: 'M',
		region: 'NLD',
		theme: 'economy'
	},
	{
		id: 'SPASTT01KRM661N',
		title: 'South Korea share prices (OECD index)',
		unit: 'Index 2015=100',
		freq: 'M',
		region: 'KOR',
		theme: 'economy'
	},
	{
		id: 'SPASTT01USM661N',
		title: 'United States share prices (OECD index)',
		unit: 'Index 2015=100',
		freq: 'M',
		region: 'USA',
		theme: 'economy'
	},
	{
		id: 'SPASTT01CNM661N',
		title: 'China share prices (OECD index)',
		unit: 'Index 2015=100',
		freq: 'M',
		region: 'CHN',
		theme: 'economy'
	},
	{ id: 'SP500', title: 'S&P 500', unit: 'Index', freq: 'D', region: 'USA', theme: 'economy' },
	{
		id: 'NASDAQCOM',
		title: 'NASDAQ Composite',
		unit: 'Index',
		freq: 'D',
		region: 'USA',
		theme: 'economy'
	},
	{
		id: 'CPIAUCSL',
		title: 'US consumer price index (CPI-U)',
		unit: 'Index 1982-84=100',
		freq: 'M',
		region: 'USA',
		theme: 'economy'
	},
	...(
		[
			['CPIUFDSL', 'food'],
			['CPIHOSSL', 'housing'],
			['CPIENGSL', 'energy'],
			['CPILFESL', 'all items less food and energy']
		] as const
	).map(([id, what]) => ({
		id,
		title: `US consumer price index (CPI-U), ${what}`,
		unit: 'Index 1982-84=100',
		freq: 'M' as const,
		region: 'USA',
		theme: 'economy'
	})),
	{
		id: 'UNRATE',
		title: 'US unemployment rate',
		unit: '%',
		freq: 'M',
		region: 'USA',
		theme: 'economy'
	},
	{
		id: 'GDPC1',
		title: 'US real GDP',
		unit: 'Billions of chained 2017 USD (SAAR)',
		freq: 'Q',
		region: 'USA',
		theme: 'economy'
	}
];

export async function fred(opts: { only?: string; job: string }) {
	await run('fred', opts.job, async (note) => {
		let total = 0;
		for (const s of FRED_SERIES) {
			if (opts.only && !opts.only.split(',').includes(s.id)) continue;
			try {
				const csv = await getText(`https://fred.stlouisfed.org/graph/fredgraph.csv?id=${s.id}`);
				const lines = csv.trim().split(/\r?\n/).slice(1);
				const rows = lines
					.map((l) => l.split(','))
					.filter(([d]) => /^\d{4}-\d{2}-\d{2}$/.test(d))
					.map(([d, v]) => ({
						series_id: s.id,
						period: d,
						value: v === '.' || v === '' ? null : Number(v)
					}));
				if (!rows.length) throw new Error('empty CSV');
				await sql.begin(async (tx) => {
					await tx`delete from stats.fred_obs where series_id = ${s.id}`;
					for (let i = 0; i < rows.length; i += 5000)
						await tx`insert into stats.fred_obs ${tx(rows.slice(i, i + 5000))}`;
				});
				const obs: ObsRow[] = rows
					.filter((r) => r.value != null)
					.map((r) => ({
						series_id: `fred:${s.id}`,
						period: r.period,
						value: r.value,
						status: null
					}));
				await writeSeries(
					[
						{
							series_id: `fred:${s.id}`,
							source_id: 'fred',
							source_code: s.id,
							title: s.title,
							unit: s.unit,
							freq: s.freq,
							region_code: s.region,
							theme: s.theme,
							meta: { url: `https://fred.stlouisfed.org/series/${s.id}` }
						}
					],
					obs
				);
				log(`fred ${s.id}: ${rows.length} rows`);
				total += rows.length;
			} catch (error) {
				note(`${s.id}: ${error instanceof Error ? error.message : String(error)}`);
			}
		}
		return total;
	});
}