Stan Runge
b566a1244f
All checks were successful
Deploy production / deploy (push) Successful in 1m20s
343 lines
6.7 KiB
TypeScript
343 lines
6.7 KiB
TypeScript
// A7 FRED graph CSV (no key). One call per series → stats.fred_obs and series `fred:<id>`.
|
||
import { getText, log, run, sql, writeSeries, type ObsRow } from './common';
|
||
|
||
type Fred = {
|
||
id: string;
|
||
title: string;
|
||
unit: string;
|
||
freq: 'D' | 'M' | 'Q' | 'A';
|
||
region: string;
|
||
theme: string;
|
||
};
|
||
export const FRED_SERIES: Fred[] = [
|
||
{
|
||
id: 'DCOILBRENTEU',
|
||
title: 'Brent crude oil spot price',
|
||
unit: 'USD per barrel',
|
||
freq: 'D',
|
||
region: 'WLD',
|
||
theme: 'energy'
|
||
},
|
||
{
|
||
id: 'PNGASEUUSDM',
|
||
title: 'EU natural gas price (TTF, IMF)',
|
||
unit: 'USD per MMBtu',
|
||
freq: 'M',
|
||
region: 'EU27',
|
||
theme: 'energy'
|
||
},
|
||
{
|
||
id: 'DFEDTARU',
|
||
title: 'Fed funds target range, upper limit',
|
||
unit: '%',
|
||
freq: 'D',
|
||
region: 'USA',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'DFEDTARL',
|
||
title: 'Fed funds target range, lower limit',
|
||
unit: '%',
|
||
freq: 'D',
|
||
region: 'USA',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'DFF',
|
||
title: 'Effective federal funds rate',
|
||
unit: '%',
|
||
freq: 'D',
|
||
region: 'USA',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'DGS10',
|
||
title: 'US Treasury 10-year yield',
|
||
unit: '%',
|
||
freq: 'D',
|
||
region: 'USA',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'DGS2',
|
||
title: 'US Treasury 2-year yield',
|
||
unit: '%',
|
||
freq: 'D',
|
||
region: 'USA',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'IRLTLT01KRM156N',
|
||
title: 'South Korea 10-year government bond yield',
|
||
unit: '%',
|
||
freq: 'M',
|
||
region: 'KOR',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'IRLTLT01FRM156N',
|
||
title: 'France 10-year government bond yield',
|
||
unit: '%',
|
||
freq: 'M',
|
||
region: 'FRA',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'IRLTLT01DEM156N',
|
||
title: 'Germany 10-year government bond yield',
|
||
unit: '%',
|
||
freq: 'M',
|
||
region: 'DEU',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'APU000072610',
|
||
title: 'US average residential electricity price',
|
||
unit: 'USD per kWh',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'energy'
|
||
},
|
||
{
|
||
id: 'APU000072620',
|
||
title: 'US average utility (piped) gas price',
|
||
unit: 'USD per therm',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'energy'
|
||
},
|
||
{
|
||
id: 'SPASTT01NLM661N',
|
||
title: 'Netherlands share prices (OECD index)',
|
||
unit: 'Index 2015=100',
|
||
freq: 'M',
|
||
region: 'NLD',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'SPASTT01KRM661N',
|
||
title: 'South Korea share prices (OECD index)',
|
||
unit: 'Index 2015=100',
|
||
freq: 'M',
|
||
region: 'KOR',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'SPASTT01USM661N',
|
||
title: 'United States share prices (OECD index)',
|
||
unit: 'Index 2015=100',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'SPASTT01CNM661N',
|
||
title: 'China share prices (OECD index)',
|
||
unit: 'Index 2015=100',
|
||
freq: 'M',
|
||
region: 'CHN',
|
||
theme: 'economy'
|
||
},
|
||
{ id: 'SP500', title: 'S&P 500', unit: 'Index', freq: 'D', region: 'USA', theme: 'economy' },
|
||
{
|
||
id: 'NASDAQCOM',
|
||
title: 'NASDAQ Composite',
|
||
unit: 'Index',
|
||
freq: 'D',
|
||
region: 'USA',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'CPIAUCSL',
|
||
title: 'US consumer price index (CPI-U)',
|
||
unit: 'Index 1982-84=100',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'economy'
|
||
},
|
||
...(
|
||
[
|
||
['CPIUFDSL', 'food'],
|
||
['CPIHOSSL', 'housing'],
|
||
['CPIENGSL', 'energy'],
|
||
['CPILFESL', 'all items less food and energy']
|
||
] as const
|
||
).map(([id, what]) => ({
|
||
id,
|
||
title: `US consumer price index (CPI-U), ${what}`,
|
||
unit: 'Index 1982-84=100',
|
||
freq: 'M' as const,
|
||
region: 'USA',
|
||
theme: 'economy'
|
||
})),
|
||
{
|
||
id: 'UNRATE',
|
||
title: 'US unemployment rate',
|
||
unit: '%',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'economy'
|
||
},
|
||
{
|
||
id: 'GDPC1',
|
||
title: 'US real GDP',
|
||
unit: 'Billions of chained 2017 USD (SAAR)',
|
||
freq: 'Q',
|
||
region: 'USA',
|
||
theme: 'economy'
|
||
},
|
||
// Jobs & cost of living
|
||
{
|
||
id: 'LNS14024887',
|
||
title: 'US unemployment rate, 16–24',
|
||
unit: '%',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'jobs'
|
||
},
|
||
{
|
||
id: 'LNS14000036',
|
||
title: 'US unemployment rate, 20–24',
|
||
unit: '%',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'jobs'
|
||
},
|
||
{
|
||
id: 'JTSJOL',
|
||
title: 'US job openings (JOLTS)',
|
||
unit: 'Thousands',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'jobs'
|
||
},
|
||
{
|
||
id: 'JTSJOR',
|
||
title: 'US job openings rate (JOLTS)',
|
||
unit: '% of jobs + openings',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'jobs'
|
||
},
|
||
{
|
||
id: 'JTSHIR',
|
||
title: 'US hires rate (JOLTS)',
|
||
unit: '%',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'jobs'
|
||
},
|
||
{
|
||
id: 'JTSQUR',
|
||
title: 'US quits rate (JOLTS)',
|
||
unit: '%',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'jobs'
|
||
},
|
||
{
|
||
id: 'UNEMPLOY',
|
||
title: 'US unemployed persons',
|
||
unit: 'Thousands',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'jobs'
|
||
},
|
||
{
|
||
id: 'LNS13025703',
|
||
title: 'US long-term unemployed (27+ weeks), share of unemployed',
|
||
unit: '%',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'jobs'
|
||
},
|
||
{
|
||
id: 'CGBD2534',
|
||
title: 'US unemployment rate, bachelor’s degree, 25–34',
|
||
unit: '%',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'jobs'
|
||
},
|
||
{
|
||
id: 'LES1252881600Q',
|
||
title: 'US median usual weekly real earnings',
|
||
unit: '1982–84 CPI-adjusted USD',
|
||
freq: 'Q',
|
||
region: 'USA',
|
||
theme: 'jobs'
|
||
},
|
||
{
|
||
id: 'LES1252881500Q',
|
||
title: 'US median usual weekly earnings, full-time',
|
||
unit: 'USD',
|
||
freq: 'Q',
|
||
region: 'USA',
|
||
theme: 'jobs'
|
||
},
|
||
{
|
||
id: 'FEDMINNFRWG',
|
||
title: 'US federal minimum wage',
|
||
unit: 'USD per hour',
|
||
freq: 'M',
|
||
region: 'USA',
|
||
theme: 'jobs'
|
||
}
|
||
];
|
||
|
||
export async function fred(opts: { only?: string; job: string }) {
|
||
await run('fred', opts.job, async (note) => {
|
||
let total = 0;
|
||
for (const s of FRED_SERIES) {
|
||
if (opts.only && !opts.only.split(',').includes(s.id)) continue;
|
||
try {
|
||
const csv = await getText(`https://fred.stlouisfed.org/graph/fredgraph.csv?id=${s.id}`);
|
||
const lines = csv.trim().split(/\r?\n/).slice(1);
|
||
const rows = lines
|
||
.map((l) => l.split(','))
|
||
.filter(([d]) => /^\d{4}-\d{2}-\d{2}$/.test(d))
|
||
.map(([d, v]) => ({
|
||
series_id: s.id,
|
||
period: d,
|
||
value: v === '.' || v === '' ? null : Number(v)
|
||
}));
|
||
if (!rows.length) throw new Error('empty CSV');
|
||
await sql.begin(async (tx) => {
|
||
await tx`delete from stats.fred_obs where series_id = ${s.id}`;
|
||
for (let i = 0; i < rows.length; i += 5000)
|
||
await tx`insert into stats.fred_obs ${tx(rows.slice(i, i + 5000))}`;
|
||
});
|
||
const obs: ObsRow[] = rows
|
||
.filter((r) => r.value != null)
|
||
.map((r) => ({
|
||
series_id: `fred:${s.id}`,
|
||
period: r.period,
|
||
value: r.value,
|
||
status: null
|
||
}));
|
||
await writeSeries(
|
||
[
|
||
{
|
||
series_id: `fred:${s.id}`,
|
||
source_id: 'fred',
|
||
source_code: s.id,
|
||
title: s.title,
|
||
unit: s.unit,
|
||
freq: s.freq,
|
||
region_code: s.region,
|
||
theme: s.theme,
|
||
meta: { url: `https://fred.stlouisfed.org/series/${s.id}` }
|
||
}
|
||
],
|
||
obs
|
||
);
|
||
log(`fred ${s.id}: ${rows.length} rows`);
|
||
total += rows.length;
|
||
} catch (error) {
|
||
note(`${s.id}: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
}
|
||
return total;
|
||
});
|
||
}
|