stats step 2: World Bank, IMF WEO, OWID, Energy-Charts, Rijksfinanciën, US Treasury, USAspending collectors + tables, timers
All checks were successful
Deploy production / deploy (push) Successful in 1m10s
All checks were successful
Deploy production / deploy (push) Successful in 1m10s
This commit is contained in:
307
scripts/stats/ustreasury.ts
Normal file
307
scripts/stats/ustreasury.ts
Normal file
@@ -0,0 +1,307 @@
|
||||
// A11 US Treasury Fiscal Data: Monthly Treasury Statement receipts (table 4) and outlays
|
||||
// (table 5), and the daily Debt to the Penny. Normalised: monthly and fiscal-year receipts by
|
||||
// type (customs duties = tariffs), outlays (total and per agency), deficit, and debt.
|
||||
import {
|
||||
getJson,
|
||||
insertChunks,
|
||||
log,
|
||||
run,
|
||||
sql,
|
||||
writeSeries,
|
||||
type ObsRow,
|
||||
type SeriesRow
|
||||
} from './common';
|
||||
|
||||
const BASE = 'https://api.fiscaldata.treasury.gov/services/api/fiscal_service';
|
||||
type Page = { data: Record<string, string>[]; meta: { 'total-pages': number } };
|
||||
|
||||
async function pullAll(endpoint: string, filter: string | null) {
|
||||
const out: Record<string, string>[] = [];
|
||||
for (let page = 1; ; page++) {
|
||||
const q = new URLSearchParams({
|
||||
'page[size]': '10000',
|
||||
'page[number]': String(page),
|
||||
sort: 'record_date'
|
||||
});
|
||||
if (filter) q.set('filter', filter);
|
||||
const body = await getJson<Page>(`${BASE}/${endpoint}?${q}`);
|
||||
out.push(...body.data);
|
||||
if (page >= (body.meta['total-pages'] ?? 1)) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
const num = (v: string | undefined) => (v == null || v === 'null' || v === '' ? null : Number(v));
|
||||
const int = (v: string | undefined) => (num(v) == null ? null : Math.trunc(Number(v)));
|
||||
|
||||
/** Receipt lines kept as series (MTS table 4 descriptions, normalised to lower case). */
|
||||
const RECEIPTS: [string, RegExp, string][] = [
|
||||
['individual', /^total -- individual income taxes$/, 'Individual income taxes'],
|
||||
['corporate', /^(total -- )?corporation income taxes$/, 'Corporation income taxes'],
|
||||
[
|
||||
'social',
|
||||
/^total -- social insurance and retirement receipts$/,
|
||||
'Social insurance and retirement receipts'
|
||||
],
|
||||
['excise', /^total -- excise taxes$/, 'Excise taxes'],
|
||||
['estate', /^(total -- )?estate and gift taxes$/, 'Estate and gift taxes'],
|
||||
['customs', /^(total -- )?customs duties$/, 'Customs duties (tariffs)'],
|
||||
['misc', /^total -- miscellaneous receipts$/, 'Miscellaneous receipts'],
|
||||
['total', /^total -- receipts$/, 'Total receipts']
|
||||
];
|
||||
const clean = (s: string) =>
|
||||
s
|
||||
.toLowerCase()
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/\s*--\s*/g, ' -- ')
|
||||
.trim();
|
||||
const slug = (s: string) =>
|
||||
s
|
||||
.toLowerCase()
|
||||
.replace(/&/g, 'and')
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '');
|
||||
/** US fiscal year of a record date (FY2026 = Oct 2025 – Sep 2026). */
|
||||
const fiscalYear = (d: string) => Number(d.slice(0, 4)) + (Number(d.slice(5, 7)) >= 10 ? 1 : 0);
|
||||
|
||||
export async function ustreasury(opts: { backfill?: boolean; job: string }) {
|
||||
await run('ustreasury', opts.job, async (note) => {
|
||||
const since = new Date(Date.now() - 400 * 86_400_000).toISOString().slice(0, 10);
|
||||
const filter = opts.backfill ? null : `record_date:gte:${since}`;
|
||||
let total = 0;
|
||||
const mts = async (
|
||||
table: 'mts_table_4' | 'mts_table_5',
|
||||
target: string,
|
||||
kind: 'rcpt' | 'outly'
|
||||
) => {
|
||||
const rows = (await pullAll(`v1/accounting/mts/${table}`, filter)).map((r) => ({
|
||||
record_date: r.record_date,
|
||||
classification_id: r.classification_id,
|
||||
parent_id: r.parent_id === 'null' ? null : r.parent_id,
|
||||
description: r.classification_desc,
|
||||
level: int(r.sequence_level_nbr),
|
||||
print_order: int(r.print_order_nbr),
|
||||
net_month: num(r[`current_month_net_${kind}_amt`]),
|
||||
net_fytd: num(r[`current_fytd_net_${kind}_amt`]),
|
||||
gross_month: num(r[`current_month_gross_${kind}_amt`])
|
||||
}));
|
||||
await sql.begin(async (tx) => {
|
||||
await insertChunks(
|
||||
tx,
|
||||
target,
|
||||
rows,
|
||||
`on conflict (record_date, classification_id) do update set
|
||||
parent_id = excluded.parent_id, description = excluded.description, level = excluded.level,
|
||||
print_order = excluded.print_order, net_month = excluded.net_month, net_fytd = excluded.net_fytd,
|
||||
gross_month = excluded.gross_month`
|
||||
);
|
||||
});
|
||||
log(`ustreasury ${table}: ${rows.length} rows`);
|
||||
return rows.length;
|
||||
};
|
||||
for (const [table, target, kind] of [
|
||||
['mts_table_4', 'ust_mts_receipts', 'rcpt'],
|
||||
['mts_table_5', 'ust_mts_outlays', 'outly']
|
||||
] as const) {
|
||||
try {
|
||||
total += await mts(table, target, kind);
|
||||
} catch (error) {
|
||||
note(`${table}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
try {
|
||||
const rows = (await pullAll('v2/accounting/od/debt_to_penny', filter)).map((r) => ({
|
||||
record_date: r.record_date,
|
||||
debt_held_public: num(r.debt_held_public_amt),
|
||||
intragov_holdings: num(r.intragov_hold_amt),
|
||||
total_debt: num(r.tot_pub_debt_out_amt)
|
||||
}));
|
||||
await sql.begin(async (tx) => {
|
||||
await insertChunks(
|
||||
tx,
|
||||
'ust_debt_daily',
|
||||
rows,
|
||||
`on conflict (record_date) do update set
|
||||
debt_held_public = excluded.debt_held_public, intragov_holdings = excluded.intragov_holdings,
|
||||
total_debt = excluded.total_debt`
|
||||
);
|
||||
});
|
||||
log(`ustreasury debt_to_penny: ${rows.length} rows`);
|
||||
total += rows.length;
|
||||
} catch (error) {
|
||||
note(`debt_to_penny: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
await normaliseUst();
|
||||
return total;
|
||||
});
|
||||
}
|
||||
|
||||
export async function normaliseUst() {
|
||||
const meta = { url: 'https://fiscaldata.treasury.gov/datasets/monthly-treasury-statement/' };
|
||||
const series: SeriesRow[] = [];
|
||||
const obs: ObsRow[] = [];
|
||||
const add = (
|
||||
id: string,
|
||||
title: string,
|
||||
unit: string,
|
||||
freq: string,
|
||||
rows: [string, number | null][],
|
||||
m = meta
|
||||
) => {
|
||||
series.push({
|
||||
series_id: id,
|
||||
source_id: 'ustreasury',
|
||||
source_code: id.slice(4),
|
||||
title,
|
||||
unit,
|
||||
freq,
|
||||
region_code: 'USA',
|
||||
theme: 'budget',
|
||||
meta: m
|
||||
});
|
||||
for (const [period, value] of rows)
|
||||
if (value != null) obs.push({ series_id: id, period, value, status: null });
|
||||
};
|
||||
const receipts = await sql<
|
||||
{
|
||||
record_date: string;
|
||||
description: string;
|
||||
net_month: number | null;
|
||||
net_fytd: number | null;
|
||||
}[]
|
||||
>`
|
||||
select record_date::text, description, net_month, net_fytd from stats.ust_mts_receipts order by record_date`;
|
||||
for (const [key, re, title] of RECEIPTS) {
|
||||
const rows = receipts.filter((r) => re.test(clean(r.description)));
|
||||
// Some months repeat a line (sub-totals); keep the first per date.
|
||||
const month = new Map<string, (typeof rows)[number]>();
|
||||
for (const r of rows) if (!month.has(r.record_date)) month.set(r.record_date, r);
|
||||
const list = [...month.values()];
|
||||
add(
|
||||
`ust:receipts:${key}:M`,
|
||||
title,
|
||||
'USD bn',
|
||||
'M',
|
||||
list.map((r) => [
|
||||
`${r.record_date.slice(0, 7)}-01`,
|
||||
r.net_month == null ? null : r.net_month / 1e9
|
||||
])
|
||||
);
|
||||
add(
|
||||
`ust:receipts:${key}:FY`,
|
||||
`${title}, fiscal year`,
|
||||
'USD bn',
|
||||
'A',
|
||||
list
|
||||
.filter((r) => r.record_date.slice(5, 7) === '09')
|
||||
.map((r) => [
|
||||
`${fiscalYear(r.record_date)}-01-01`,
|
||||
r.net_fytd == null ? null : r.net_fytd / 1e9
|
||||
])
|
||||
);
|
||||
}
|
||||
const outlays = await sql<
|
||||
{
|
||||
record_date: string;
|
||||
description: string;
|
||||
level: number | null;
|
||||
net_month: number | null;
|
||||
net_fytd: number | null;
|
||||
}[]
|
||||
>`
|
||||
select record_date::text, description, level, net_month, net_fytd from stats.ust_mts_outlays order by record_date, print_order`;
|
||||
const pick = (re: RegExp) => {
|
||||
const m = new Map<string, (typeof outlays)[number]>();
|
||||
for (const r of outlays)
|
||||
if (re.test(clean(r.description)) && !m.has(r.record_date)) m.set(r.record_date, r);
|
||||
return [...m.values()];
|
||||
};
|
||||
const totalOut = pick(/^total outlays$/);
|
||||
add(
|
||||
'ust:outlays:total:M',
|
||||
'Total outlays',
|
||||
'USD bn',
|
||||
'M',
|
||||
totalOut.map((r) => [
|
||||
`${r.record_date.slice(0, 7)}-01`,
|
||||
r.net_month == null ? null : r.net_month / 1e9
|
||||
])
|
||||
);
|
||||
add(
|
||||
'ust:outlays:total:FY',
|
||||
'Total outlays, fiscal year',
|
||||
'USD bn',
|
||||
'A',
|
||||
totalOut
|
||||
.filter((r) => r.record_date.slice(5, 7) === '09')
|
||||
.map((r) => [
|
||||
`${fiscalYear(r.record_date)}-01-01`,
|
||||
r.net_fytd == null ? null : r.net_fytd / 1e9
|
||||
])
|
||||
);
|
||||
const deficit = pick(/^total surplus \(\+\) or deficit \(-\)$/);
|
||||
add(
|
||||
'ust:deficit:M',
|
||||
'Surplus (+) or deficit (−)',
|
||||
'USD bn',
|
||||
'M',
|
||||
deficit.map((r) => [
|
||||
`${r.record_date.slice(0, 7)}-01`,
|
||||
r.net_month == null ? null : r.net_month / 1e9
|
||||
])
|
||||
);
|
||||
add(
|
||||
'ust:deficit:FY',
|
||||
'Surplus (+) or deficit (−), fiscal year',
|
||||
'USD bn',
|
||||
'A',
|
||||
deficit
|
||||
.filter((r) => r.record_date.slice(5, 7) === '09')
|
||||
.map((r) => [
|
||||
`${fiscalYear(r.record_date)}-01-01`,
|
||||
r.net_fytd == null ? null : r.net_fytd / 1e9
|
||||
])
|
||||
);
|
||||
// Agency totals: "Total--Department of Defense--Military Programs" etc.
|
||||
const agencies = new Map<string, Map<string, number | null>>();
|
||||
for (const r of outlays) {
|
||||
const d = r.description.replace(/\s+/g, ' ').trim();
|
||||
const m = /^Total\s*--\s*(.+)$/.exec(d);
|
||||
if (!m || r.level !== 2 || r.record_date.slice(5, 7) !== '09') continue;
|
||||
const name = m[1].trim();
|
||||
if (!agencies.has(name)) agencies.set(name, new Map());
|
||||
const fy = `${fiscalYear(r.record_date)}-01-01`;
|
||||
if (!agencies.get(name)!.has(fy))
|
||||
agencies.get(name)!.set(fy, r.net_fytd == null ? null : r.net_fytd / 1e9);
|
||||
}
|
||||
for (const [name, years] of agencies)
|
||||
add(
|
||||
`ust:outlays:${slug(name)}:FY`,
|
||||
`Outlays: ${name}, fiscal year`,
|
||||
'USD bn',
|
||||
'A',
|
||||
[...years.entries()],
|
||||
{ ...meta, agency: name }
|
||||
);
|
||||
const debt = await sql<
|
||||
{ d: string; total_debt: number | null; debt_held_public: number | null }[]
|
||||
>`
|
||||
select record_date::text as d, total_debt, debt_held_public from stats.ust_debt_daily order by 1`;
|
||||
const dm = { url: 'https://fiscaldata.treasury.gov/datasets/debt-to-the-penny/' };
|
||||
add(
|
||||
'ust:debt:D',
|
||||
'Total public debt outstanding',
|
||||
'USD tn',
|
||||
'D',
|
||||
debt.map((r) => [r.d, r.total_debt == null ? null : r.total_debt / 1e12]),
|
||||
dm
|
||||
);
|
||||
add(
|
||||
'ust:debt:public:D',
|
||||
'Debt held by the public',
|
||||
'USD tn',
|
||||
'D',
|
||||
debt.map((r) => [r.d, r.debt_held_public == null ? null : r.debt_held_public / 1e12]),
|
||||
dm
|
||||
);
|
||||
return writeSeries(series, obs);
|
||||
}
|
||||
Reference in New Issue
Block a user