stats step 2: World Bank, IMF WEO, OWID, Energy-Charts, Rijksfinanciën, US Treasury, USAspending collectors + tables, timers

scripts/stats/usaspending.ts

@@ -0,0 +1,86 @@
+// A12 USAspending: budgetary resources, obligations and outlays per top-tier agency and fiscal
+// year (one call for the agency list, one per agency for its history). Small: ~110 agencies.
+import {
+	getJson,
+	insertChunks,
+	log,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+
+const BASE = 'https://api.usaspending.gov/api/v2';
+type Agency = { toptier_code: string; agency_name: string; abbreviation: string | null };
+type Resources = {
+	agency_data_by_year: {
+		fiscal_year: number;
+		agency_budgetary_resources: number | null;
+		agency_total_obligated: number | null;
+		agency_total_outlayed: number | null;
+	}[];
+};
+
+export async function usaspending(opts: { job: string }) {
+	await run('usaspending', opts.job, async (note) => {
+		const { results } = await getJson<{ results: Agency[] }>(
+			`${BASE}/references/toptier_agencies/`
+		);
+		const rows: Record<string, unknown>[] = [];
+		for (const a of results) {
+			try {
+				const r = await getJson<Resources>(`${BASE}/agency/${a.toptier_code}/budgetary_resources/`);
+				for (const y of r.agency_data_by_year ?? [])
+					rows.push({
+						toptier_code: a.toptier_code,
+						fiscal_year: y.fiscal_year,
+						agency_name: a.agency_name,
+						abbreviation: a.abbreviation,
+						budgetary_resources: y.agency_budgetary_resources,
+						obligated: y.agency_total_obligated,
+						outlayed: y.agency_total_outlayed
+					});
+			} catch (error) {
+				note(`${a.toptier_code}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		if (!rows.length) throw new Error('no rows');
+		await sql.begin(async (tx) => {
+			await tx`delete from stats.usas_agency_fy`;
+			await insertChunks(tx, 'usas_agency_fy', rows);
+		});
+		const series: SeriesRow[] = [];
+		const obs: ObsRow[] = [];
+		for (const a of results) {
+			const id = `usas:outlays:${a.toptier_code}`;
+			const mine = rows.filter((r) => r.toptier_code === a.toptier_code);
+			if (!mine.length) continue;
+			series.push({
+				series_id: id,
+				source_id: 'usaspending',
+				source_code: a.toptier_code,
+				title: `Outlays: ${a.agency_name}`,
+				unit: 'USD bn',
+				freq: 'A',
+				region_code: 'USA',
+				theme: 'budget',
+				meta: {
+					abbreviation: a.abbreviation,
+					url: `https://www.usaspending.gov/agency/${a.toptier_code}`
+				}
+			});
+			for (const r of mine)
+				if (r.outlayed != null)
+					obs.push({
+						series_id: id,
+						period: `${r.fiscal_year}-01-01`,
+						value: Number(r.outlayed) / 1e9,
+						status: null
+					});
+		}
+		await writeSeries(series, obs);
+		log(`usaspending: ${rows.length} agency-years`);
+		return rows.length;
+	});
+}