stan/website
1
0
Fork 0
Code Issues 13 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

stats: new public stats schema (source/series/obs/region/fetch_run + raw CBS, Eurostat, FRED, BIS tables) and the stats collector (scripts/stats-fetch.ts); preflight checks the stats tables
Some checks failed
Deploy production / deploy (push) Failing after 54s
Details

Browse Source
This commit is contained in:
Stan Runge
2026-10-25 20:27:04 +00:00
parent 902b1cadd5
commit 8ef291f586
14 changed files with 33128 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
scripts/stats/fred.ts Normal file
View File

@@ -0,0 +1,230 @@
// A7 FRED graph CSV (no key). One call per series → stats.fred_obs and series `fred:<id>`.
import { getText, log, run, sql, writeSeries, type ObsRow } from './common';
type Fred = {
id: string;
title: string;
unit: string;
freq: 'D' | 'M' | 'Q' | 'A';
region: string;
theme: string;
};
export const FRED_SERIES: Fred[] = [
{
id: 'DCOILBRENTEU',
title: 'Brent crude oil spot price',
unit: 'USD per barrel',
freq: 'D',
region: 'WLD',
theme: 'energy'
},
{
id: 'PNGASEUUSDM',
title: 'EU natural gas price (TTF, IMF)',
unit: 'USD per MMBtu',
freq: 'M',
region: 'EU27',
theme: 'energy'
},
{
id: 'DFEDTARU',
title: 'Fed funds target range, upper limit',
unit: '%',
freq: 'D',
region: 'USA',
theme: 'economy'
},
{
id: 'DFEDTARL',
title: 'Fed funds target range, lower limit',
unit: '%',
freq: 'D',
region: 'USA',
theme: 'economy'
},
{
id: 'DFF',
title: 'Effective federal funds rate',
unit: '%',
freq: 'D',
region: 'USA',
theme: 'economy'
},
{
id: 'DGS10',
title: 'US Treasury 10-year yield',
unit: '%',
freq: 'D',
region: 'USA',
theme: 'economy'
},
{
id: 'DGS2',
title: 'US Treasury 2-year yield',
unit: '%',
freq: 'D',
region: 'USA',
theme: 'economy'
},
{
id: 'IRLTLT01KRM156N',
title: 'South Korea 10-year government bond yield',
unit: '%',
freq: 'M',
region: 'KOR',
theme: 'economy'
},
{
id: 'IRLTLT01FRM156N',
title: 'France 10-year government bond yield',
unit: '%',
freq: 'M',
region: 'FRA',
theme: 'economy'
},
{
id: 'IRLTLT01DEM156N',
title: 'Germany 10-year government bond yield',
unit: '%',
freq: 'M',
region: 'DEU',
theme: 'economy'
},
{
id: 'APU000072610',
title: 'US average residential electricity price',
unit: 'USD per kWh',
freq: 'M',
region: 'USA',
theme: 'energy'
},
{
id: 'APU000072620',
title: 'US average utility (piped) gas price',
unit: 'USD per therm',
freq: 'M',
region: 'USA',
theme: 'energy'
},
{
id: 'SPASTT01NLM661N',
title: 'Netherlands share prices (OECD index)',
unit: 'Index 2015=100',
freq: 'M',
region: 'NLD',
theme: 'economy'
},
{
id: 'SPASTT01KRM661N',
title: 'South Korea share prices (OECD index)',
unit: 'Index 2015=100',
freq: 'M',
region: 'KOR',
theme: 'economy'
},
{
id: 'SPASTT01USM661N',
title: 'United States share prices (OECD index)',
unit: 'Index 2015=100',
freq: 'M',
region: 'USA',
theme: 'economy'
},
{
id: 'SPASTT01CNM661N',
title: 'China share prices (OECD index)',
unit: 'Index 2015=100',
freq: 'M',
region: 'CHN',
theme: 'economy'
},
{ id: 'SP500', title: 'S&P 500', unit: 'Index', freq: 'D', region: 'USA', theme: 'economy' },
{
id: 'NASDAQCOM',
title: 'NASDAQ Composite',
unit: 'Index',
freq: 'D',
region: 'USA',
theme: 'economy'
},
{
id: 'CPIAUCSL',
title: 'US consumer price index (CPI-U)',
unit: 'Index 1982-84=100',
freq: 'M',
region: 'USA',
theme: 'economy'
},
{
id: 'UNRATE',
title: 'US unemployment rate',
unit: '%',
freq: 'M',
region: 'USA',
theme: 'economy'
},
{
id: 'GDPC1',
title: 'US real GDP',
unit: 'Billions of chained 2017 USD (SAAR)',
freq: 'Q',
region: 'USA',
theme: 'economy'
}
];
export async function fred(opts: { only?: string; job: string }) {
await run('fred', opts.job, async (note) => {
let total = 0;
for (const s of FRED_SERIES) {
if (opts.only && !opts.only.split(',').includes(s.id)) continue;
try {
const csv = await getText(`https://fred.stlouisfed.org/graph/fredgraph.csv?id=${s.id}`);
const lines = csv.trim().split(/\r?\n/).slice(1);
const rows = lines
.map((l) => l.split(','))
.filter(([d]) => /^\d{4}-\d{2}-\d{2}$/.test(d))
.map(([d, v]) => ({
series_id: s.id,
period: d,
value: v === '.' || v === '' ? null : Number(v)
}));
if (!rows.length) throw new Error('empty CSV');
await sql.begin(async (tx) => {
await tx`delete from stats.fred_obs where series_id = ${s.id}`;
for (let i = 0; i < rows.length; i += 5000)
await tx`insert into stats.fred_obs ${tx(rows.slice(i, i + 5000))}`;
});
const obs: ObsRow[] = rows
.filter((r) => r.value != null)
.map((r) => ({
series_id: `fred:${s.id}`,
period: r.period,
value: r.value,
status: null
}));
await writeSeries(
[
{
series_id: `fred:${s.id}`,
source_id: 'fred',
source_code: s.id,
title: s.title,
unit: s.unit,
freq: s.freq,
region_code: s.region,
theme: s.theme,
meta: { url: `https://fred.stlouisfed.org/series/${s.id}` }
}
],
obs
);
log(`fred ${s.id}: ${rows.length} rows`);
total += rows.length;
} catch (error) {
note(`${s.id}: ${error instanceof Error ? error.message : String(error)}`);
}
}
return total;
});
}