Stan Runge
8958d3ff8f
All checks were successful
Deploy production / deploy (push) Successful in 1m10s
308 lines
9.5 KiB
TypeScript
308 lines
9.5 KiB
TypeScript
// A11 US Treasury Fiscal Data: Monthly Treasury Statement receipts (table 4) and outlays
|
||
// (table 5), and the daily Debt to the Penny. Normalised: monthly and fiscal-year receipts by
|
||
// type (customs duties = tariffs), outlays (total and per agency), deficit, and debt.
|
||
import {
|
||
getJson,
|
||
insertChunks,
|
||
log,
|
||
run,
|
||
sql,
|
||
writeSeries,
|
||
type ObsRow,
|
||
type SeriesRow
|
||
} from './common';
|
||
|
||
const BASE = 'https://api.fiscaldata.treasury.gov/services/api/fiscal_service';
|
||
type Page = { data: Record<string, string>[]; meta: { 'total-pages': number } };
|
||
|
||
async function pullAll(endpoint: string, filter: string | null) {
|
||
const out: Record<string, string>[] = [];
|
||
for (let page = 1; ; page++) {
|
||
const q = new URLSearchParams({
|
||
'page[size]': '10000',
|
||
'page[number]': String(page),
|
||
sort: 'record_date'
|
||
});
|
||
if (filter) q.set('filter', filter);
|
||
const body = await getJson<Page>(`${BASE}/${endpoint}?${q}`);
|
||
out.push(...body.data);
|
||
if (page >= (body.meta['total-pages'] ?? 1)) break;
|
||
}
|
||
return out;
|
||
}
|
||
const num = (v: string | undefined) => (v == null || v === 'null' || v === '' ? null : Number(v));
|
||
const int = (v: string | undefined) => (num(v) == null ? null : Math.trunc(Number(v)));
|
||
|
||
/** Receipt lines kept as series (MTS table 4 descriptions, normalised to lower case). */
|
||
const RECEIPTS: [string, RegExp, string][] = [
|
||
['individual', /^total -- individual income taxes$/, 'Individual income taxes'],
|
||
['corporate', /^(total -- )?corporation income taxes$/, 'Corporation income taxes'],
|
||
[
|
||
'social',
|
||
/^total -- social insurance and retirement receipts$/,
|
||
'Social insurance and retirement receipts'
|
||
],
|
||
['excise', /^total -- excise taxes$/, 'Excise taxes'],
|
||
['estate', /^(total -- )?estate and gift taxes$/, 'Estate and gift taxes'],
|
||
['customs', /^(total -- )?customs duties$/, 'Customs duties (tariffs)'],
|
||
['misc', /^total -- miscellaneous receipts$/, 'Miscellaneous receipts'],
|
||
['total', /^total -- receipts$/, 'Total receipts']
|
||
];
|
||
const clean = (s: string) =>
|
||
s
|
||
.toLowerCase()
|
||
.replace(/\s+/g, ' ')
|
||
.replace(/\s*--\s*/g, ' -- ')
|
||
.trim();
|
||
const slug = (s: string) =>
|
||
s
|
||
.toLowerCase()
|
||
.replace(/&/g, 'and')
|
||
.replace(/[^a-z0-9]+/g, '-')
|
||
.replace(/^-|-$/g, '');
|
||
/** US fiscal year of a record date (FY2026 = Oct 2025 – Sep 2026). */
|
||
const fiscalYear = (d: string) => Number(d.slice(0, 4)) + (Number(d.slice(5, 7)) >= 10 ? 1 : 0);
|
||
|
||
export async function ustreasury(opts: { backfill?: boolean; job: string }) {
|
||
await run('ustreasury', opts.job, async (note) => {
|
||
const since = new Date(Date.now() - 400 * 86_400_000).toISOString().slice(0, 10);
|
||
const filter = opts.backfill ? null : `record_date:gte:${since}`;
|
||
let total = 0;
|
||
const mts = async (
|
||
table: 'mts_table_4' | 'mts_table_5',
|
||
target: string,
|
||
kind: 'rcpt' | 'outly'
|
||
) => {
|
||
const rows = (await pullAll(`v1/accounting/mts/${table}`, filter)).map((r) => ({
|
||
record_date: r.record_date,
|
||
classification_id: r.classification_id,
|
||
parent_id: r.parent_id === 'null' ? null : r.parent_id,
|
||
description: r.classification_desc,
|
||
level: int(r.sequence_level_nbr),
|
||
print_order: int(r.print_order_nbr),
|
||
net_month: num(r[`current_month_net_${kind}_amt`]),
|
||
net_fytd: num(r[`current_fytd_net_${kind}_amt`]),
|
||
gross_month: num(r[`current_month_gross_${kind}_amt`])
|
||
}));
|
||
await sql.begin(async (tx) => {
|
||
await insertChunks(
|
||
tx,
|
||
target,
|
||
rows,
|
||
`on conflict (record_date, classification_id) do update set
|
||
parent_id = excluded.parent_id, description = excluded.description, level = excluded.level,
|
||
print_order = excluded.print_order, net_month = excluded.net_month, net_fytd = excluded.net_fytd,
|
||
gross_month = excluded.gross_month`
|
||
);
|
||
});
|
||
log(`ustreasury ${table}: ${rows.length} rows`);
|
||
return rows.length;
|
||
};
|
||
for (const [table, target, kind] of [
|
||
['mts_table_4', 'ust_mts_receipts', 'rcpt'],
|
||
['mts_table_5', 'ust_mts_outlays', 'outly']
|
||
] as const) {
|
||
try {
|
||
total += await mts(table, target, kind);
|
||
} catch (error) {
|
||
note(`${table}: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
}
|
||
try {
|
||
const rows = (await pullAll('v2/accounting/od/debt_to_penny', filter)).map((r) => ({
|
||
record_date: r.record_date,
|
||
debt_held_public: num(r.debt_held_public_amt),
|
||
intragov_holdings: num(r.intragov_hold_amt),
|
||
total_debt: num(r.tot_pub_debt_out_amt)
|
||
}));
|
||
await sql.begin(async (tx) => {
|
||
await insertChunks(
|
||
tx,
|
||
'ust_debt_daily',
|
||
rows,
|
||
`on conflict (record_date) do update set
|
||
debt_held_public = excluded.debt_held_public, intragov_holdings = excluded.intragov_holdings,
|
||
total_debt = excluded.total_debt`
|
||
);
|
||
});
|
||
log(`ustreasury debt_to_penny: ${rows.length} rows`);
|
||
total += rows.length;
|
||
} catch (error) {
|
||
note(`debt_to_penny: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
await normaliseUst();
|
||
return total;
|
||
});
|
||
}
|
||
|
||
export async function normaliseUst() {
|
||
const meta = { url: 'https://fiscaldata.treasury.gov/datasets/monthly-treasury-statement/' };
|
||
const series: SeriesRow[] = [];
|
||
const obs: ObsRow[] = [];
|
||
const add = (
|
||
id: string,
|
||
title: string,
|
||
unit: string,
|
||
freq: string,
|
||
rows: [string, number | null][],
|
||
m = meta
|
||
) => {
|
||
series.push({
|
||
series_id: id,
|
||
source_id: 'ustreasury',
|
||
source_code: id.slice(4),
|
||
title,
|
||
unit,
|
||
freq,
|
||
region_code: 'USA',
|
||
theme: 'budget',
|
||
meta: m
|
||
});
|
||
for (const [period, value] of rows)
|
||
if (value != null) obs.push({ series_id: id, period, value, status: null });
|
||
};
|
||
const receipts = await sql<
|
||
{
|
||
record_date: string;
|
||
description: string;
|
||
net_month: number | null;
|
||
net_fytd: number | null;
|
||
}[]
|
||
>`
|
||
select record_date::text, description, net_month, net_fytd from stats.ust_mts_receipts order by record_date`;
|
||
for (const [key, re, title] of RECEIPTS) {
|
||
const rows = receipts.filter((r) => re.test(clean(r.description)));
|
||
// Some months repeat a line (sub-totals); keep the first per date.
|
||
const month = new Map<string, (typeof rows)[number]>();
|
||
for (const r of rows) if (!month.has(r.record_date)) month.set(r.record_date, r);
|
||
const list = [...month.values()];
|
||
add(
|
||
`ust:receipts:${key}:M`,
|
||
title,
|
||
'USD bn',
|
||
'M',
|
||
list.map((r) => [
|
||
`${r.record_date.slice(0, 7)}-01`,
|
||
r.net_month == null ? null : r.net_month / 1e9
|
||
])
|
||
);
|
||
add(
|
||
`ust:receipts:${key}:FY`,
|
||
`${title}, fiscal year`,
|
||
'USD bn',
|
||
'A',
|
||
list
|
||
.filter((r) => r.record_date.slice(5, 7) === '09')
|
||
.map((r) => [
|
||
`${fiscalYear(r.record_date)}-01-01`,
|
||
r.net_fytd == null ? null : r.net_fytd / 1e9
|
||
])
|
||
);
|
||
}
|
||
const outlays = await sql<
|
||
{
|
||
record_date: string;
|
||
description: string;
|
||
level: number | null;
|
||
net_month: number | null;
|
||
net_fytd: number | null;
|
||
}[]
|
||
>`
|
||
select record_date::text, description, level, net_month, net_fytd from stats.ust_mts_outlays order by record_date, print_order`;
|
||
const pick = (re: RegExp) => {
|
||
const m = new Map<string, (typeof outlays)[number]>();
|
||
for (const r of outlays)
|
||
if (re.test(clean(r.description)) && !m.has(r.record_date)) m.set(r.record_date, r);
|
||
return [...m.values()];
|
||
};
|
||
const totalOut = pick(/^total outlays$/);
|
||
add(
|
||
'ust:outlays:total:M',
|
||
'Total outlays',
|
||
'USD bn',
|
||
'M',
|
||
totalOut.map((r) => [
|
||
`${r.record_date.slice(0, 7)}-01`,
|
||
r.net_month == null ? null : r.net_month / 1e9
|
||
])
|
||
);
|
||
add(
|
||
'ust:outlays:total:FY',
|
||
'Total outlays, fiscal year',
|
||
'USD bn',
|
||
'A',
|
||
totalOut
|
||
.filter((r) => r.record_date.slice(5, 7) === '09')
|
||
.map((r) => [
|
||
`${fiscalYear(r.record_date)}-01-01`,
|
||
r.net_fytd == null ? null : r.net_fytd / 1e9
|
||
])
|
||
);
|
||
const deficit = pick(/^total surplus \(\+\) or deficit \(-\)$/);
|
||
add(
|
||
'ust:deficit:M',
|
||
'Surplus (+) or deficit (−)',
|
||
'USD bn',
|
||
'M',
|
||
deficit.map((r) => [
|
||
`${r.record_date.slice(0, 7)}-01`,
|
||
r.net_month == null ? null : r.net_month / 1e9
|
||
])
|
||
);
|
||
add(
|
||
'ust:deficit:FY',
|
||
'Surplus (+) or deficit (−), fiscal year',
|
||
'USD bn',
|
||
'A',
|
||
deficit
|
||
.filter((r) => r.record_date.slice(5, 7) === '09')
|
||
.map((r) => [
|
||
`${fiscalYear(r.record_date)}-01-01`,
|
||
r.net_fytd == null ? null : r.net_fytd / 1e9
|
||
])
|
||
);
|
||
// Agency totals: "Total--Department of Defense--Military Programs" etc.
|
||
const agencies = new Map<string, Map<string, number | null>>();
|
||
for (const r of outlays) {
|
||
const d = r.description.replace(/\s+/g, ' ').trim();
|
||
const m = /^Total\s*--\s*(.+)$/.exec(d);
|
||
if (!m || r.level !== 2 || r.record_date.slice(5, 7) !== '09') continue;
|
||
const name = m[1].trim();
|
||
if (!agencies.has(name)) agencies.set(name, new Map());
|
||
const fy = `${fiscalYear(r.record_date)}-01-01`;
|
||
if (!agencies.get(name)!.has(fy))
|
||
agencies.get(name)!.set(fy, r.net_fytd == null ? null : r.net_fytd / 1e9);
|
||
}
|
||
for (const [name, years] of agencies)
|
||
add(
|
||
`ust:outlays:${slug(name)}:FY`,
|
||
`Outlays: ${name}, fiscal year`,
|
||
'USD bn',
|
||
'A',
|
||
[...years.entries()],
|
||
{ ...meta, agency: name }
|
||
);
|
||
const debt = await sql<
|
||
{ d: string; total_debt: number | null; debt_held_public: number | null }[]
|
||
>`
|
||
select record_date::text as d, total_debt, debt_held_public from stats.ust_debt_daily order by 1`;
|
||
const dm = { url: 'https://fiscaldata.treasury.gov/datasets/debt-to-the-penny/' };
|
||
add(
|
||
'ust:debt:D',
|
||
'Total public debt outstanding',
|
||
'USD tn',
|
||
'D',
|
||
debt.map((r) => [r.d, r.total_debt == null ? null : r.total_debt / 1e12]),
|
||
dm
|
||
);
|
||
add(
|
||
'ust:debt:public:D',
|
||
'Debt held by the public',
|
||
'USD tn',
|
||
'D',
|
||
debt.map((r) => [r.d, r.debt_held_public == null ? null : r.debt_held_public / 1e12]),
|
||
dm
|
||
);
|
||
return writeSeries(series, obs);
|
||
}
|