stats: jobs & cost of living theme (unemployment by age, vacancies, JOLTS, graduates, job quality, wages, price levels, housing), grad view cards, Beveridge curve; jobs collector (OECD SDMX, Big Mac index, derived series) + Eurostat/World Bank/FRED jobs series, stats-jobs.timer

2026-10-26 18:07:56 +00:00
parent f2f7bd0a19
commit b566a1244f
13 changed files with 1492 additions and 30 deletions
--- a/scripts/stats/jobs.ts
+++ b/scripts/stats/jobs.ts
@@ -0,0 +1,498 @@
+// Jobs & cost of living (/stats/<place>/jobs), the parts that are not a plain Eurostat / World
+// Bank / FRED series list:
+//   - OECD Data Explorer SDMX (no key): average and minimum wages in USD PPP, hours, long hours,
+//     temporary-job shares, long-term unemployment, graduate employment and the tertiary
+//     earnings premium (Education at a Glance), house price-to-income / price-to-rent, monthly
+//     youth unemployment and comparative price levels (USA = 100). Series `oecd:jobs:<name>:<ISO3>`.
+//   - Derived from those: real wage growth (yearly change of the constant-price average wage) and
+//     an approximate median full-time wage (real minimum wage ÷ minimum-to-median ratio),
+//     `jobs:realwagegrowth:<ISO3>` and `jobs:medianwage:<ISO3>`.
+//   - The Economist Big Mac index (GitHub CSV, CC BY 4.0): `economist:bigmac:<ISO3>:<measure>`.
+//   - Derived from FRED: US job openings per unemployed person (JTSJOL / UNEMPLOY).
+//   - Derived from World Bank: price level of private consumption, USA = 100
+//     (PA.NUS.PRVT.PP ÷ PA.NUS.FCRF; WDI dropped PA.NUS.PPPC.RF), `jobs:pricelevel:<ISO3>`.
+// Numbeo is not used: it has no free API and its terms forbid scraping.
+import {
+	getText,
+	http,
+	log,
+	parsePeriod,
+	REGIONS,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+
+const OECD = 'https://sdmx.oecd.org/public/rest/data';
+const CSV = { accept: 'application/vnd.sdmx.data+csv; charset=utf-8' };
+/** Countries and aggregates asked from OECD; non-members simply come back empty. */
+const AREAS = [
+	'NLD',
+	'USA',
+	'KOR',
+	'JPN',
+	'CAN',
+	'CHN',
+	'DEU',
+	'FRA',
+	'BEL',
+	'GBR',
+	'IND',
+	'BRA',
+	'EU27_2020',
+	'EA20',
+	'OECD'
+];
+const AREA_REGION: Record<string, string> = { EU27_2020: 'EU27', EA20: 'EA', OECD: 'OECD' };
+const areaRegion = (a: string) => AREA_REGION[a] ?? a;
+
+type OecdSpec = {
+	job: string;
+	/** Dataflow `agency,id,version`. */
+	flow: string;
+	/** Key after the area, e.g. `.WG.USD_PPP.A.Q.MEAN._Z` (the area list is prefixed). */
+	key: string;
+	start?: string;
+	/** Row → our series name (or null to skip). */
+	pick: (r: Record<string, string>) => string | null;
+	meta: Record<string, { title: string; unit: string }>;
+};
+
+const SPECS: OecdSpec[] = [
+	{
+		job: 'wages',
+		flow: 'OECD.ELS.SAE,DSD_EARNINGS@AV_AN_WAGE,1.0',
+		key: '.WG.USD_PPP.A.Q.MEAN._Z',
+		pick: () => 'avwage',
+		meta: {
+			avwage: {
+				title: 'Average annual wage, full-time equivalent (constant prices, USD PPP)',
+				unit: 'USD PPP'
+			}
+		}
+	},
+	{
+		job: 'wages',
+		flow: 'OECD.ELS.SAE,DSD_EARNINGS@RMW,1.0',
+		key: '.SM_WG.USD_PPP.A+H.Q._Z._Z',
+		pick: (r) => (r.PAY_PERIOD === 'A' ? 'minwage' : r.PAY_PERIOD === 'H' ? 'minwage_h' : null),
+		meta: {
+			minwage: { title: 'Real statutory minimum wage, yearly (USD PPP)', unit: 'USD PPP' },
+			minwage_h: { title: 'Real statutory minimum wage, hourly (USD PPP)', unit: 'USD PPP / h' }
+		}
+	},
+	{
+		job: 'wages',
+		flow: 'OECD.ELS.SAE,DSD_EARNINGS@MIN2AVE,1.0',
+		key: '.MR_WG_FT.PT_WG_A_SAL_FT._Z._Z.MEDIAN._Z',
+		pick: () => 'min2median',
+		meta: {
+			min2median: {
+				title: 'Minimum wage relative to the median full-time wage',
+				unit: '% of median'
+			}
+		}
+	},
+	{
+		job: 'quality',
+		flow: 'OECD.ELS.SAE,DSD_HW@DF_AVG_ANN_HRS_WKD,1.0',
+		key: '.HW.H_Y_PS._Z._Z.EMP.A.ACTUAL._T._Z.MEAN._Z._T',
+		pick: () => 'hours',
+		meta: { hours: { title: 'Average annual hours actually worked per worker', unit: 'hours' } }
+	},
+	{
+		job: 'quality',
+		flow: 'OECD.ELS.SAE,DSD_HW@DF_EMP_LNG_USL_WK_HRS,1.0',
+		key: '.SH_EMP_H.PT_WR_SEX_AGE_EMP_ST._T.Y15T64.EMP.W.USUAL._T._Z.OBS.H_GE50.MAIN',
+		pick: () => 'longhours',
+		meta: {
+			longhours: {
+				title: 'Employees and self-employed usually working 50+ hours a week',
+				unit: '% of employed'
+			}
+		}
+	},
+	{
+		job: 'quality',
+		flow: 'OECD.ELS.SAE,DSD_TEMP@DF_TEMP_I,1.0',
+		key: '.EMP_TEMP.PT_POP_SUB._T._T+Y15T24.ICSE93_1.A',
+		pick: (r) => (r.AGE === '_T' ? 'temp' : 'temp_youth'),
+		meta: {
+			temp: { title: 'Temporary employment, share of employees', unit: '% of employees' },
+			temp_youth: {
+				title: 'Temporary employment, share of employees aged 15–24',
+				unit: '% of employees'
+			}
+		}
+	},
+	{
+		job: 'finding',
+		flow: 'OECD.ELS.SAE,DSD_DUR@DF_DUR_I,1.0',
+		key: '.UNEMP.PT_POP_SUB._T._T.Y_GE1.A',
+		pick: () => 'ltu',
+		meta: {
+			ltu: { title: 'Long-term unemployment (1 year or more)', unit: '% of unemployed' }
+		}
+	},
+	{
+		job: 'finding',
+		flow: 'OECD.SDD.TPS,DSD_LFS@DF_IALFS_UNE_M,1.0',
+		key: '.UNE_LF_M.PT_LF_SUB._Z.Y._T.Y15T24+Y_GE25+Y_GE15._Z.M',
+		start: '1990-01',
+		pick: (r) =>
+			r.AGE === 'Y15T24'
+				? 'unemp_youth'
+				: r.AGE === 'Y_GE25'
+					? 'unemp_25plus'
+					: r.AGE === 'Y_GE15'
+						? 'unemp'
+						: null,
+		meta: {
+			unemp: { title: 'Unemployment rate, monthly (OECD harmonised)', unit: '%' },
+			unemp_youth: { title: 'Youth unemployment rate 15–24, monthly', unit: '%' },
+			unemp_25plus: { title: 'Unemployment rate 25+, monthly', unit: '%' }
+		}
+	},
+	{
+		job: 'grads',
+		flow: 'OECD.EDU.IMEP,DSD_EAG_LSO_EA@DF_LSO_NEAC_EMP,1.0',
+		key: '._T.Y25T34.ISCED11A_5T8._T.EMPLOYMENT._Z._T._Z.ED_NED.EMP._Z.PT_POP_SUB.OBS._Z.NEAC.A',
+		pick: () => 'grademp',
+		meta: {
+			grademp: {
+				title: 'Employment rate of tertiary-educated 25–34 year-olds',
+				unit: '% of age group'
+			}
+		}
+	},
+	{
+		job: 'grads',
+		flow: 'OECD.EDU.IMEP,DSD_EAG_LSO_EA@DF_LSO_EARN_REL_UPPER,1.0',
+		key: '._T.Y25T34.ISCED11A_5T8._T.EARN_RELATIVE._Z._T._Z.ED_NED.EMP._Z.PT_EARN_WR_ISCED11A_3.OBS.FT.EARNINGS.A',
+		pick: () => 'premium',
+		meta: {
+			premium: {
+				title:
+					'Earnings of tertiary-educated 25–34 year-olds (full-time), relative to upper secondary',
+				unit: 'upper secondary = 100'
+			}
+		}
+	},
+	{
+		job: 'housing',
+		flow: 'OECD.ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES,1.0',
+		key: '.Q.HPI_YDH_AVG+HPI_RPI_AVG+RPI.',
+		start: '1970-Q1',
+		pick: (r) =>
+			r.MEASURE === 'HPI_YDH_AVG'
+				? 'ptoi'
+				: r.MEASURE === 'HPI_RPI_AVG'
+					? 'ptor'
+					: r.MEASURE === 'RPI'
+						? 'rent'
+						: null,
+		meta: {
+			ptoi: {
+				title: 'House price-to-income ratio, vs its long-term average',
+				unit: 'long-term avg = 100'
+			},
+			ptor: {
+				title: 'House price-to-rent ratio, vs its long-term average',
+				unit: 'long-term avg = 100'
+			},
+			rent: { title: 'Rent price index', unit: 'Index 2015=100' }
+		}
+	},
+	{
+		job: 'prices',
+		flow: 'OECD.SDD.TPS,DSD_PPP_M@DF_PP_CPL_M,1.0',
+		key: '.M.CPL.IX..USA',
+		start: '2015-01',
+		pick: (r) => (r.COUNTERPART_AREA === 'USA' ? 'cpl' : null),
+		meta: {
+			cpl: {
+				title: 'Comparative price level of final consumption (USA = 100)',
+				unit: 'USA = 100'
+			}
+		}
+	}
+];
+
+function parseCsv(text: string): Record<string, string>[] {
+	const lines = text.trim().split(/\r?\n/);
+	const head = lines[0].split(',');
+	return lines.slice(1).map((l) => Object.fromEntries(l.split(',').map((v, i) => [head[i], v])));
+}
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+async function pullOecd(spec: OecdSpec): Promise<number> {
+	const url = `${OECD}/${spec.flow}/${AREAS.join('+')}${spec.key}?startPeriod=${spec.start ?? '1990'}`;
+	const res = await http(url, { headers: CSV });
+	const rows = parseCsv(await res.text());
+	const series = new Map<string, SeriesRow>();
+	const obs: ObsRow[] = [];
+	for (const r of rows) {
+		const name = spec.pick(r);
+		if (!name || r.OBS_VALUE === '' || r.OBS_VALUE == null) continue;
+		const value = Number(r.OBS_VALUE);
+		const p = parsePeriod(r.TIME_PERIOD ?? '');
+		if (!p || !Number.isFinite(value)) continue;
+		const region = areaRegion(r.REF_AREA);
+		const id = `oecd:jobs:${name}:${region}`;
+		if (!series.has(id)) {
+			const m = spec.meta[name];
+			series.set(id, {
+				series_id: id,
+				source_id: 'oecd',
+				source_code: `${spec.flow}/${r.REF_AREA}${spec.key}`,
+				title: m.title,
+				unit: m.unit,
+				freq: p.freq,
+				region_code: region,
+				theme: 'jobs',
+				meta: { url: 'https://data-explorer.oecd.org/', flow: spec.flow }
+			});
+		}
+		obs.push({ series_id: id, period: p.date, value, status: r.OBS_STATUS || null });
+	}
+	if (!obs.length) throw new Error(`no rows for ${spec.flow}`);
+	// One row per (series, period): SDMX can repeat a period with a different status.
+	const seen = new Set<string>();
+	const unique = obs.filter((o) => {
+		const k = `${o.series_id}|${o.period}`;
+		if (seen.has(k)) return false;
+		seen.add(k);
+		return true;
+	});
+	await writeSeries([...series.values()], unique);
+	log(`jobs oecd ${spec.flow.split(',')[1]}: ${unique.length} obs, ${series.size} series`);
+	return unique.length;
+}
+
+// ── Big Mac index ─────────────────────────────────────────────────────────────────────────
+const BIGMAC =
+	'https://raw.githubusercontent.com/TheEconomist/big-mac-data/master/output-data/big-mac-full-index.csv';
+const BIGMAC_REGION: Record<string, string> = { EUZ: 'EA' };
+
+async function bigmac(): Promise<number> {
+	const rows = parseCsv(await getText(BIGMAC));
+	const known = new Set(REGIONS.map((r) => r.region_code));
+	const measures: Record<string, { title: string; unit: string }> = {
+		dollar_price: { title: 'Big Mac price in US dollars', unit: 'US$' },
+		USD_raw: {
+			title: 'Big Mac index: local currency vs the US dollar (raw)',
+			unit: '% over/under-valued'
+		}
+	};
+	const series = new Map<string, SeriesRow>();
+	const obs: ObsRow[] = [];
+	for (const r of rows) {
+		const region = BIGMAC_REGION[r.iso_a3] ?? r.iso_a3;
+		if (!known.has(region) || !/^\d{4}-\d{2}-\d{2}$/.test(r.date)) continue;
+		for (const [m, meta] of Object.entries(measures)) {
+			const raw = Number(r[m]);
+			if (r[m] === '' || !Number.isFinite(raw)) continue;
+			const id = `economist:bigmac:${region}:${m}`;
+			if (!series.has(id))
+				series.set(id, {
+					series_id: id,
+					source_id: 'economist',
+					source_code: `big-mac-full-index/${r.iso_a3}/${m}`,
+					title: meta.title,
+					unit: meta.unit,
+					freq: 'S',
+					region_code: region,
+					theme: 'jobs',
+					meta: { url: 'https://github.com/TheEconomist/big-mac-data' }
+				});
+			obs.push({
+				series_id: id,
+				period: r.date,
+				value: m === 'USD_raw' ? Math.round(raw * 1000) / 10 : raw,
+				status: null
+			});
+		}
+	}
+	if (!obs.length) throw new Error('Big Mac CSV: no rows');
+	await writeSeries([...series.values()], obs);
+	log(`jobs bigmac: ${obs.length} obs, ${series.size} series`);
+	return obs.length;
+}
+
+// ── Derived: US openings per unemployed person ─────────────────────────────────────────────
+async function joltsRatio(): Promise<number> {
+	const rows = await sql<{ period: string; value: number }[]>`
+		select o.period::text as period, round((o.value / u.value)::numeric, 3)::float8 as value
+		from stats.obs o join stats.obs u on u.period = o.period and u.series_id = 'fred:UNEMPLOY'
+		where o.series_id = 'fred:JTSJOL' and u.value > 0 and o.value is not null
+		order by o.period`;
+	if (!rows.length) throw new Error('JOLTS ratio: run the fred collector first (JTSJOL, UNEMPLOY)');
+	const id = 'jobs:openings_per_unemployed:USA';
+	await writeSeries(
+		[
+			{
+				series_id: id,
+				source_id: 'fred',
+				source_code: 'JTSJOL/UNEMPLOY',
+				title: 'US job openings per unemployed person (JOLTS openings ÷ unemployed)',
+				unit: 'openings per unemployed',
+				freq: 'M',
+				region_code: 'USA',
+				theme: 'jobs',
+				meta: { url: 'https://fred.stlouisfed.org/series/JTSJOL', derived: 'JTSJOL / UNEMPLOY' }
+			}
+		],
+		rows.map((r) => ({ series_id: id, period: r.period, value: r.value, status: null }))
+	);
+	return rows.length;
+}
+
+// ── Derived: real wage growth and an approximate median wage ────────────────────────────────
+async function wageDerived(): Promise<number> {
+	const growth = await sql<{ region: string; period: string; value: number }[]>`
+		select s.region_code as region, o.period::text as period,
+			round((100 * (o.value / p.value - 1))::numeric, 2)::float8 as value
+		from stats.series s
+		join stats.obs o on o.series_id = s.series_id
+		join stats.obs p on p.series_id = s.series_id and p.period = o.period - interval '1 year'
+		where s.series_id like 'oecd:jobs:avwage:%' and p.value > 0 and o.value is not null
+		order by 1, 2`;
+	const median = await sql<{ region: string; period: string; value: number }[]>`
+		select s.region_code as region, o.period::text as period,
+			round((o.value / (r.value / 100))::numeric, 0)::float8 as value
+		from stats.series s
+		join stats.obs o on o.series_id = s.series_id
+		join stats.obs r on r.series_id = 'oecd:jobs:min2median:' || s.region_code and r.period = o.period
+		where s.series_id like 'oecd:jobs:minwage:%' and r.value > 0 and o.value is not null
+		order by 1, 2`;
+	const write = async (
+		rows: { region: string; period: string; value: number }[],
+		name: string,
+		title: string,
+		unit: string,
+		derived: string
+	) => {
+		const regions = [...new Set(rows.map((r) => r.region))];
+		await writeSeries(
+			regions.map((region) => ({
+				series_id: `jobs:${name}:${region}`,
+				source_id: 'oecd',
+				source_code: derived,
+				title,
+				unit,
+				freq: 'A',
+				region_code: region,
+				theme: 'jobs',
+				meta: { url: 'https://data-explorer.oecd.org/', derived }
+			})),
+			rows.map((r) => ({
+				series_id: `jobs:${name}:${r.region}`,
+				period: r.period,
+				value: r.value,
+				status: null
+			}))
+		);
+		return rows.length;
+	};
+	if (!growth.length) throw new Error('wage derived: run the wages job first');
+	return (
+		(await write(
+			growth,
+			'realwagegrowth',
+			'Real average wage growth (constant prices)',
+			'%',
+			'yoy(AV_AN_WAGE, constant prices)'
+		)) +
+		(await write(
+			median,
+			'medianwage',
+			'Median full-time wage, approx. (real minimum wage ÷ minimum-to-median ratio)',
+			'USD PPP',
+			'RMW / MIN2AVE(median)'
+		))
+	);
+}
+
+// ── Derived: price level (USA = 100) from the World Bank PPP and exchange rate ─────────────
+async function priceLevel(): Promise<number> {
+	const rows = await sql<{ region: string; period: string; value: number }[]>`
+		select p.region_code as region, o.period::text as period,
+			round((100 * o.value / x.value)::numeric, 1)::float8 as value
+		from stats.series p
+		join stats.obs o on o.series_id = p.series_id
+		join stats.obs x on x.period = o.period and x.series_id = 'wb:PA.NUS.FCRF:' || p.region_code
+		where p.series_id like 'wb:PA.NUS.PRVT.PP:%' and p.region_code is not null
+			and x.value > 0 and o.value is not null
+		order by 1, 2`;
+	if (!rows.length)
+		throw new Error('price level: run worldbank first (PA.NUS.PRVT.PP, PA.NUS.FCRF)');
+	const regions = [...new Set(rows.map((r) => r.region))];
+	await writeSeries(
+		regions.map((region) => ({
+			series_id: `jobs:pricelevel:${region}`,
+			source_id: 'worldbank',
+			source_code: 'PA.NUS.PRVT.PP/PA.NUS.FCRF',
+			title: 'Price level of household consumption (PPP ÷ exchange rate, USA = 100)',
+			unit: 'USA = 100',
+			freq: 'A',
+			region_code: region,
+			theme: 'jobs',
+			meta: {
+				url: 'https://data.worldbank.org/indicator/PA.NUS.PRVT.PP',
+				derived: '100 × PA.NUS.PRVT.PP / PA.NUS.FCRF'
+			}
+		})),
+		rows.map((r) => ({
+			series_id: `jobs:pricelevel:${r.region}`,
+			period: r.period,
+			value: r.value,
+			status: null
+		}))
+	);
+	return rows.length;
+}
+
+export async function jobs(opts: { only?: string; job: string }) {
+	await run('oecd', `jobs:${opts.job}`, async (note) => {
+		const want = (j: string) => !opts.only || opts.only.split(',').includes(j);
+		let total = 0;
+		for (const spec of SPECS) {
+			if (!want(spec.job)) continue;
+			try {
+				total += await pullOecd(spec);
+			} catch (error) {
+				note(`${spec.flow}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+			await sleep(3200); // OECD: ≤ 20 requests a minute
+		}
+		if (want('bigmac'))
+			try {
+				total += await bigmac();
+			} catch (error) {
+				note(`bigmac: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		if (want('jolts'))
+			try {
+				total += await joltsRatio();
+			} catch (error) {
+				note(`jolts: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		if (want('wages') || want('derived'))
+			try {
+				total += await wageDerived();
+			} catch (error) {
+				note(`wage derived: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		if (want('pricelevel'))
+			try {
+				total += await priceLevel();
+			} catch (error) {
+				note(`pricelevel: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		if (!total) throw new Error('nothing loaded');
+		return total;
+	});
+}