Stan Runge
b566a1244f
All checks were successful
Deploy production / deploy (push) Successful in 1m20s
499 lines
16 KiB
TypeScript
499 lines
16 KiB
TypeScript
// Jobs & cost of living (/stats/<place>/jobs), the parts that are not a plain Eurostat / World
|
||
// Bank / FRED series list:
|
||
// - OECD Data Explorer SDMX (no key): average and minimum wages in USD PPP, hours, long hours,
|
||
// temporary-job shares, long-term unemployment, graduate employment and the tertiary
|
||
// earnings premium (Education at a Glance), house price-to-income / price-to-rent, monthly
|
||
// youth unemployment and comparative price levels (USA = 100). Series `oecd:jobs:<name>:<ISO3>`.
|
||
// - Derived from those: real wage growth (yearly change of the constant-price average wage) and
|
||
// an approximate median full-time wage (real minimum wage ÷ minimum-to-median ratio),
|
||
// `jobs:realwagegrowth:<ISO3>` and `jobs:medianwage:<ISO3>`.
|
||
// - The Economist Big Mac index (GitHub CSV, CC BY 4.0): `economist:bigmac:<ISO3>:<measure>`.
|
||
// - Derived from FRED: US job openings per unemployed person (JTSJOL / UNEMPLOY).
|
||
// - Derived from World Bank: price level of private consumption, USA = 100
|
||
// (PA.NUS.PRVT.PP ÷ PA.NUS.FCRF; WDI dropped PA.NUS.PPPC.RF), `jobs:pricelevel:<ISO3>`.
|
||
// Numbeo is not used: it has no free API and its terms forbid scraping.
|
||
import {
|
||
getText,
|
||
http,
|
||
log,
|
||
parsePeriod,
|
||
REGIONS,
|
||
run,
|
||
sql,
|
||
writeSeries,
|
||
type ObsRow,
|
||
type SeriesRow
|
||
} from './common';
|
||
|
||
const OECD = 'https://sdmx.oecd.org/public/rest/data';
|
||
const CSV = { accept: 'application/vnd.sdmx.data+csv; charset=utf-8' };
|
||
/** Countries and aggregates asked from OECD; non-members simply come back empty. */
|
||
const AREAS = [
|
||
'NLD',
|
||
'USA',
|
||
'KOR',
|
||
'JPN',
|
||
'CAN',
|
||
'CHN',
|
||
'DEU',
|
||
'FRA',
|
||
'BEL',
|
||
'GBR',
|
||
'IND',
|
||
'BRA',
|
||
'EU27_2020',
|
||
'EA20',
|
||
'OECD'
|
||
];
|
||
const AREA_REGION: Record<string, string> = { EU27_2020: 'EU27', EA20: 'EA', OECD: 'OECD' };
|
||
const areaRegion = (a: string) => AREA_REGION[a] ?? a;
|
||
|
||
type OecdSpec = {
|
||
job: string;
|
||
/** Dataflow `agency,id,version`. */
|
||
flow: string;
|
||
/** Key after the area, e.g. `.WG.USD_PPP.A.Q.MEAN._Z` (the area list is prefixed). */
|
||
key: string;
|
||
start?: string;
|
||
/** Row → our series name (or null to skip). */
|
||
pick: (r: Record<string, string>) => string | null;
|
||
meta: Record<string, { title: string; unit: string }>;
|
||
};
|
||
|
||
const SPECS: OecdSpec[] = [
|
||
{
|
||
job: 'wages',
|
||
flow: 'OECD.ELS.SAE,DSD_EARNINGS@AV_AN_WAGE,1.0',
|
||
key: '.WG.USD_PPP.A.Q.MEAN._Z',
|
||
pick: () => 'avwage',
|
||
meta: {
|
||
avwage: {
|
||
title: 'Average annual wage, full-time equivalent (constant prices, USD PPP)',
|
||
unit: 'USD PPP'
|
||
}
|
||
}
|
||
},
|
||
{
|
||
job: 'wages',
|
||
flow: 'OECD.ELS.SAE,DSD_EARNINGS@RMW,1.0',
|
||
key: '.SM_WG.USD_PPP.A+H.Q._Z._Z',
|
||
pick: (r) => (r.PAY_PERIOD === 'A' ? 'minwage' : r.PAY_PERIOD === 'H' ? 'minwage_h' : null),
|
||
meta: {
|
||
minwage: { title: 'Real statutory minimum wage, yearly (USD PPP)', unit: 'USD PPP' },
|
||
minwage_h: { title: 'Real statutory minimum wage, hourly (USD PPP)', unit: 'USD PPP / h' }
|
||
}
|
||
},
|
||
{
|
||
job: 'wages',
|
||
flow: 'OECD.ELS.SAE,DSD_EARNINGS@MIN2AVE,1.0',
|
||
key: '.MR_WG_FT.PT_WG_A_SAL_FT._Z._Z.MEDIAN._Z',
|
||
pick: () => 'min2median',
|
||
meta: {
|
||
min2median: {
|
||
title: 'Minimum wage relative to the median full-time wage',
|
||
unit: '% of median'
|
||
}
|
||
}
|
||
},
|
||
{
|
||
job: 'quality',
|
||
flow: 'OECD.ELS.SAE,DSD_HW@DF_AVG_ANN_HRS_WKD,1.0',
|
||
key: '.HW.H_Y_PS._Z._Z.EMP.A.ACTUAL._T._Z.MEAN._Z._T',
|
||
pick: () => 'hours',
|
||
meta: { hours: { title: 'Average annual hours actually worked per worker', unit: 'hours' } }
|
||
},
|
||
{
|
||
job: 'quality',
|
||
flow: 'OECD.ELS.SAE,DSD_HW@DF_EMP_LNG_USL_WK_HRS,1.0',
|
||
key: '.SH_EMP_H.PT_WR_SEX_AGE_EMP_ST._T.Y15T64.EMP.W.USUAL._T._Z.OBS.H_GE50.MAIN',
|
||
pick: () => 'longhours',
|
||
meta: {
|
||
longhours: {
|
||
title: 'Employees and self-employed usually working 50+ hours a week',
|
||
unit: '% of employed'
|
||
}
|
||
}
|
||
},
|
||
{
|
||
job: 'quality',
|
||
flow: 'OECD.ELS.SAE,DSD_TEMP@DF_TEMP_I,1.0',
|
||
key: '.EMP_TEMP.PT_POP_SUB._T._T+Y15T24.ICSE93_1.A',
|
||
pick: (r) => (r.AGE === '_T' ? 'temp' : 'temp_youth'),
|
||
meta: {
|
||
temp: { title: 'Temporary employment, share of employees', unit: '% of employees' },
|
||
temp_youth: {
|
||
title: 'Temporary employment, share of employees aged 15–24',
|
||
unit: '% of employees'
|
||
}
|
||
}
|
||
},
|
||
{
|
||
job: 'finding',
|
||
flow: 'OECD.ELS.SAE,DSD_DUR@DF_DUR_I,1.0',
|
||
key: '.UNEMP.PT_POP_SUB._T._T.Y_GE1.A',
|
||
pick: () => 'ltu',
|
||
meta: {
|
||
ltu: { title: 'Long-term unemployment (1 year or more)', unit: '% of unemployed' }
|
||
}
|
||
},
|
||
{
|
||
job: 'finding',
|
||
flow: 'OECD.SDD.TPS,DSD_LFS@DF_IALFS_UNE_M,1.0',
|
||
key: '.UNE_LF_M.PT_LF_SUB._Z.Y._T.Y15T24+Y_GE25+Y_GE15._Z.M',
|
||
start: '1990-01',
|
||
pick: (r) =>
|
||
r.AGE === 'Y15T24'
|
||
? 'unemp_youth'
|
||
: r.AGE === 'Y_GE25'
|
||
? 'unemp_25plus'
|
||
: r.AGE === 'Y_GE15'
|
||
? 'unemp'
|
||
: null,
|
||
meta: {
|
||
unemp: { title: 'Unemployment rate, monthly (OECD harmonised)', unit: '%' },
|
||
unemp_youth: { title: 'Youth unemployment rate 15–24, monthly', unit: '%' },
|
||
unemp_25plus: { title: 'Unemployment rate 25+, monthly', unit: '%' }
|
||
}
|
||
},
|
||
{
|
||
job: 'grads',
|
||
flow: 'OECD.EDU.IMEP,DSD_EAG_LSO_EA@DF_LSO_NEAC_EMP,1.0',
|
||
key: '._T.Y25T34.ISCED11A_5T8._T.EMPLOYMENT._Z._T._Z.ED_NED.EMP._Z.PT_POP_SUB.OBS._Z.NEAC.A',
|
||
pick: () => 'grademp',
|
||
meta: {
|
||
grademp: {
|
||
title: 'Employment rate of tertiary-educated 25–34 year-olds',
|
||
unit: '% of age group'
|
||
}
|
||
}
|
||
},
|
||
{
|
||
job: 'grads',
|
||
flow: 'OECD.EDU.IMEP,DSD_EAG_LSO_EA@DF_LSO_EARN_REL_UPPER,1.0',
|
||
key: '._T.Y25T34.ISCED11A_5T8._T.EARN_RELATIVE._Z._T._Z.ED_NED.EMP._Z.PT_EARN_WR_ISCED11A_3.OBS.FT.EARNINGS.A',
|
||
pick: () => 'premium',
|
||
meta: {
|
||
premium: {
|
||
title:
|
||
'Earnings of tertiary-educated 25–34 year-olds (full-time), relative to upper secondary',
|
||
unit: 'upper secondary = 100'
|
||
}
|
||
}
|
||
},
|
||
{
|
||
job: 'housing',
|
||
flow: 'OECD.ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES,1.0',
|
||
key: '.Q.HPI_YDH_AVG+HPI_RPI_AVG+RPI.',
|
||
start: '1970-Q1',
|
||
pick: (r) =>
|
||
r.MEASURE === 'HPI_YDH_AVG'
|
||
? 'ptoi'
|
||
: r.MEASURE === 'HPI_RPI_AVG'
|
||
? 'ptor'
|
||
: r.MEASURE === 'RPI'
|
||
? 'rent'
|
||
: null,
|
||
meta: {
|
||
ptoi: {
|
||
title: 'House price-to-income ratio, vs its long-term average',
|
||
unit: 'long-term avg = 100'
|
||
},
|
||
ptor: {
|
||
title: 'House price-to-rent ratio, vs its long-term average',
|
||
unit: 'long-term avg = 100'
|
||
},
|
||
rent: { title: 'Rent price index', unit: 'Index 2015=100' }
|
||
}
|
||
},
|
||
{
|
||
job: 'prices',
|
||
flow: 'OECD.SDD.TPS,DSD_PPP_M@DF_PP_CPL_M,1.0',
|
||
key: '.M.CPL.IX..USA',
|
||
start: '2015-01',
|
||
pick: (r) => (r.COUNTERPART_AREA === 'USA' ? 'cpl' : null),
|
||
meta: {
|
||
cpl: {
|
||
title: 'Comparative price level of final consumption (USA = 100)',
|
||
unit: 'USA = 100'
|
||
}
|
||
}
|
||
}
|
||
];
|
||
|
||
function parseCsv(text: string): Record<string, string>[] {
|
||
const lines = text.trim().split(/\r?\n/);
|
||
const head = lines[0].split(',');
|
||
return lines.slice(1).map((l) => Object.fromEntries(l.split(',').map((v, i) => [head[i], v])));
|
||
}
|
||
|
||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||
|
||
async function pullOecd(spec: OecdSpec): Promise<number> {
|
||
const url = `${OECD}/${spec.flow}/${AREAS.join('+')}${spec.key}?startPeriod=${spec.start ?? '1990'}`;
|
||
const res = await http(url, { headers: CSV });
|
||
const rows = parseCsv(await res.text());
|
||
const series = new Map<string, SeriesRow>();
|
||
const obs: ObsRow[] = [];
|
||
for (const r of rows) {
|
||
const name = spec.pick(r);
|
||
if (!name || r.OBS_VALUE === '' || r.OBS_VALUE == null) continue;
|
||
const value = Number(r.OBS_VALUE);
|
||
const p = parsePeriod(r.TIME_PERIOD ?? '');
|
||
if (!p || !Number.isFinite(value)) continue;
|
||
const region = areaRegion(r.REF_AREA);
|
||
const id = `oecd:jobs:${name}:${region}`;
|
||
if (!series.has(id)) {
|
||
const m = spec.meta[name];
|
||
series.set(id, {
|
||
series_id: id,
|
||
source_id: 'oecd',
|
||
source_code: `${spec.flow}/${r.REF_AREA}${spec.key}`,
|
||
title: m.title,
|
||
unit: m.unit,
|
||
freq: p.freq,
|
||
region_code: region,
|
||
theme: 'jobs',
|
||
meta: { url: 'https://data-explorer.oecd.org/', flow: spec.flow }
|
||
});
|
||
}
|
||
obs.push({ series_id: id, period: p.date, value, status: r.OBS_STATUS || null });
|
||
}
|
||
if (!obs.length) throw new Error(`no rows for ${spec.flow}`);
|
||
// One row per (series, period): SDMX can repeat a period with a different status.
|
||
const seen = new Set<string>();
|
||
const unique = obs.filter((o) => {
|
||
const k = `${o.series_id}|${o.period}`;
|
||
if (seen.has(k)) return false;
|
||
seen.add(k);
|
||
return true;
|
||
});
|
||
await writeSeries([...series.values()], unique);
|
||
log(`jobs oecd ${spec.flow.split(',')[1]}: ${unique.length} obs, ${series.size} series`);
|
||
return unique.length;
|
||
}
|
||
|
||
// ── Big Mac index ─────────────────────────────────────────────────────────────────────────
|
||
const BIGMAC =
|
||
'https://raw.githubusercontent.com/TheEconomist/big-mac-data/master/output-data/big-mac-full-index.csv';
|
||
const BIGMAC_REGION: Record<string, string> = { EUZ: 'EA' };
|
||
|
||
async function bigmac(): Promise<number> {
|
||
const rows = parseCsv(await getText(BIGMAC));
|
||
const known = new Set(REGIONS.map((r) => r.region_code));
|
||
const measures: Record<string, { title: string; unit: string }> = {
|
||
dollar_price: { title: 'Big Mac price in US dollars', unit: 'US$' },
|
||
USD_raw: {
|
||
title: 'Big Mac index: local currency vs the US dollar (raw)',
|
||
unit: '% over/under-valued'
|
||
}
|
||
};
|
||
const series = new Map<string, SeriesRow>();
|
||
const obs: ObsRow[] = [];
|
||
for (const r of rows) {
|
||
const region = BIGMAC_REGION[r.iso_a3] ?? r.iso_a3;
|
||
if (!known.has(region) || !/^\d{4}-\d{2}-\d{2}$/.test(r.date)) continue;
|
||
for (const [m, meta] of Object.entries(measures)) {
|
||
const raw = Number(r[m]);
|
||
if (r[m] === '' || !Number.isFinite(raw)) continue;
|
||
const id = `economist:bigmac:${region}:${m}`;
|
||
if (!series.has(id))
|
||
series.set(id, {
|
||
series_id: id,
|
||
source_id: 'economist',
|
||
source_code: `big-mac-full-index/${r.iso_a3}/${m}`,
|
||
title: meta.title,
|
||
unit: meta.unit,
|
||
freq: 'S',
|
||
region_code: region,
|
||
theme: 'jobs',
|
||
meta: { url: 'https://github.com/TheEconomist/big-mac-data' }
|
||
});
|
||
obs.push({
|
||
series_id: id,
|
||
period: r.date,
|
||
value: m === 'USD_raw' ? Math.round(raw * 1000) / 10 : raw,
|
||
status: null
|
||
});
|
||
}
|
||
}
|
||
if (!obs.length) throw new Error('Big Mac CSV: no rows');
|
||
await writeSeries([...series.values()], obs);
|
||
log(`jobs bigmac: ${obs.length} obs, ${series.size} series`);
|
||
return obs.length;
|
||
}
|
||
|
||
// ── Derived: US openings per unemployed person ─────────────────────────────────────────────
|
||
async function joltsRatio(): Promise<number> {
|
||
const rows = await sql<{ period: string; value: number }[]>`
|
||
select o.period::text as period, round((o.value / u.value)::numeric, 3)::float8 as value
|
||
from stats.obs o join stats.obs u on u.period = o.period and u.series_id = 'fred:UNEMPLOY'
|
||
where o.series_id = 'fred:JTSJOL' and u.value > 0 and o.value is not null
|
||
order by o.period`;
|
||
if (!rows.length) throw new Error('JOLTS ratio: run the fred collector first (JTSJOL, UNEMPLOY)');
|
||
const id = 'jobs:openings_per_unemployed:USA';
|
||
await writeSeries(
|
||
[
|
||
{
|
||
series_id: id,
|
||
source_id: 'fred',
|
||
source_code: 'JTSJOL/UNEMPLOY',
|
||
title: 'US job openings per unemployed person (JOLTS openings ÷ unemployed)',
|
||
unit: 'openings per unemployed',
|
||
freq: 'M',
|
||
region_code: 'USA',
|
||
theme: 'jobs',
|
||
meta: { url: 'https://fred.stlouisfed.org/series/JTSJOL', derived: 'JTSJOL / UNEMPLOY' }
|
||
}
|
||
],
|
||
rows.map((r) => ({ series_id: id, period: r.period, value: r.value, status: null }))
|
||
);
|
||
return rows.length;
|
||
}
|
||
|
||
// ── Derived: real wage growth and an approximate median wage ────────────────────────────────
|
||
async function wageDerived(): Promise<number> {
|
||
const growth = await sql<{ region: string; period: string; value: number }[]>`
|
||
select s.region_code as region, o.period::text as period,
|
||
round((100 * (o.value / p.value - 1))::numeric, 2)::float8 as value
|
||
from stats.series s
|
||
join stats.obs o on o.series_id = s.series_id
|
||
join stats.obs p on p.series_id = s.series_id and p.period = o.period - interval '1 year'
|
||
where s.series_id like 'oecd:jobs:avwage:%' and p.value > 0 and o.value is not null
|
||
order by 1, 2`;
|
||
const median = await sql<{ region: string; period: string; value: number }[]>`
|
||
select s.region_code as region, o.period::text as period,
|
||
round((o.value / (r.value / 100))::numeric, 0)::float8 as value
|
||
from stats.series s
|
||
join stats.obs o on o.series_id = s.series_id
|
||
join stats.obs r on r.series_id = 'oecd:jobs:min2median:' || s.region_code and r.period = o.period
|
||
where s.series_id like 'oecd:jobs:minwage:%' and r.value > 0 and o.value is not null
|
||
order by 1, 2`;
|
||
const write = async (
|
||
rows: { region: string; period: string; value: number }[],
|
||
name: string,
|
||
title: string,
|
||
unit: string,
|
||
derived: string
|
||
) => {
|
||
const regions = [...new Set(rows.map((r) => r.region))];
|
||
await writeSeries(
|
||
regions.map((region) => ({
|
||
series_id: `jobs:${name}:${region}`,
|
||
source_id: 'oecd',
|
||
source_code: derived,
|
||
title,
|
||
unit,
|
||
freq: 'A',
|
||
region_code: region,
|
||
theme: 'jobs',
|
||
meta: { url: 'https://data-explorer.oecd.org/', derived }
|
||
})),
|
||
rows.map((r) => ({
|
||
series_id: `jobs:${name}:${r.region}`,
|
||
period: r.period,
|
||
value: r.value,
|
||
status: null
|
||
}))
|
||
);
|
||
return rows.length;
|
||
};
|
||
if (!growth.length) throw new Error('wage derived: run the wages job first');
|
||
return (
|
||
(await write(
|
||
growth,
|
||
'realwagegrowth',
|
||
'Real average wage growth (constant prices)',
|
||
'%',
|
||
'yoy(AV_AN_WAGE, constant prices)'
|
||
)) +
|
||
(await write(
|
||
median,
|
||
'medianwage',
|
||
'Median full-time wage, approx. (real minimum wage ÷ minimum-to-median ratio)',
|
||
'USD PPP',
|
||
'RMW / MIN2AVE(median)'
|
||
))
|
||
);
|
||
}
|
||
|
||
// ── Derived: price level (USA = 100) from the World Bank PPP and exchange rate ─────────────
|
||
async function priceLevel(): Promise<number> {
|
||
const rows = await sql<{ region: string; period: string; value: number }[]>`
|
||
select p.region_code as region, o.period::text as period,
|
||
round((100 * o.value / x.value)::numeric, 1)::float8 as value
|
||
from stats.series p
|
||
join stats.obs o on o.series_id = p.series_id
|
||
join stats.obs x on x.period = o.period and x.series_id = 'wb:PA.NUS.FCRF:' || p.region_code
|
||
where p.series_id like 'wb:PA.NUS.PRVT.PP:%' and p.region_code is not null
|
||
and x.value > 0 and o.value is not null
|
||
order by 1, 2`;
|
||
if (!rows.length)
|
||
throw new Error('price level: run worldbank first (PA.NUS.PRVT.PP, PA.NUS.FCRF)');
|
||
const regions = [...new Set(rows.map((r) => r.region))];
|
||
await writeSeries(
|
||
regions.map((region) => ({
|
||
series_id: `jobs:pricelevel:${region}`,
|
||
source_id: 'worldbank',
|
||
source_code: 'PA.NUS.PRVT.PP/PA.NUS.FCRF',
|
||
title: 'Price level of household consumption (PPP ÷ exchange rate, USA = 100)',
|
||
unit: 'USA = 100',
|
||
freq: 'A',
|
||
region_code: region,
|
||
theme: 'jobs',
|
||
meta: {
|
||
url: 'https://data.worldbank.org/indicator/PA.NUS.PRVT.PP',
|
||
derived: '100 × PA.NUS.PRVT.PP / PA.NUS.FCRF'
|
||
}
|
||
})),
|
||
rows.map((r) => ({
|
||
series_id: `jobs:pricelevel:${r.region}`,
|
||
period: r.period,
|
||
value: r.value,
|
||
status: null
|
||
}))
|
||
);
|
||
return rows.length;
|
||
}
|
||
|
||
export async function jobs(opts: { only?: string; job: string }) {
|
||
await run('oecd', `jobs:${opts.job}`, async (note) => {
|
||
const want = (j: string) => !opts.only || opts.only.split(',').includes(j);
|
||
let total = 0;
|
||
for (const spec of SPECS) {
|
||
if (!want(spec.job)) continue;
|
||
try {
|
||
total += await pullOecd(spec);
|
||
} catch (error) {
|
||
note(`${spec.flow}: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
await sleep(3200); // OECD: ≤ 20 requests a minute
|
||
}
|
||
if (want('bigmac'))
|
||
try {
|
||
total += await bigmac();
|
||
} catch (error) {
|
||
note(`bigmac: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
if (want('jolts'))
|
||
try {
|
||
total += await joltsRatio();
|
||
} catch (error) {
|
||
note(`jolts: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
if (want('wages') || want('derived'))
|
||
try {
|
||
total += await wageDerived();
|
||
} catch (error) {
|
||
note(`wage derived: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
if (want('pricelevel'))
|
||
try {
|
||
total += await priceLevel();
|
||
} catch (error) {
|
||
note(`pricelevel: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
if (!total) throw new Error('nothing loaded');
|
||
return total;
|
||
});
|
||
}
|