stats step 2: World Bank, IMF WEO, OWID, Energy-Charts, Rijksfinanciën, US Treasury, USAspending collectors + tables, timers
All checks were successful
Deploy production / deploy (push) Successful in 1m10s
All checks were successful
Deploy production / deploy (push) Successful in 1m10s
This commit is contained in:
218
scripts/stats/owid.ts
Normal file
218
scripts/stats/owid.ts
Normal file
@@ -0,0 +1,218 @@
|
||||
// A9 Our World in Data: Maddison GDP per capita (NL back to 1348) and the OWID energy dataset
|
||||
// (energy mix per country). Both are full CSV downloads, replaced on every run.
|
||||
import {
|
||||
COMPARE_ISO3,
|
||||
getText,
|
||||
insertChunks,
|
||||
log,
|
||||
run,
|
||||
sql,
|
||||
writeSeries,
|
||||
type ObsRow,
|
||||
type SeriesRow
|
||||
} from './common';
|
||||
|
||||
const MADDISON_URL =
|
||||
'https://ourworldindata.org/grapher/gdp-per-capita-maddison-project-database.csv?csvType=full&useColumnShortNames=true';
|
||||
const ENERGY_URL = 'https://owid-public.owid.io/data/energy/owid-energy-data.csv';
|
||||
|
||||
/** OWID energy columns kept (raw) and the ones normalised into series for compared countries. */
|
||||
export const ENERGY_COLUMNS: Record<string, { title: string; unit: string; series?: boolean }> = {
|
||||
population: { title: 'Population', unit: 'people' },
|
||||
energy_per_capita: { title: 'Primary energy per person', unit: 'kWh', series: true },
|
||||
primary_energy_consumption: { title: 'Primary energy consumption', unit: 'TWh', series: true },
|
||||
per_capita_electricity: { title: 'Electricity generation per person', unit: 'kWh', series: true },
|
||||
electricity_generation: { title: 'Electricity generation', unit: 'TWh' },
|
||||
carbon_intensity_elec: {
|
||||
title: 'Carbon intensity of electricity',
|
||||
unit: 'gCO₂/kWh',
|
||||
series: true
|
||||
},
|
||||
fossil_share_energy: { title: 'Fossil fuels, share of primary energy', unit: '%', series: true },
|
||||
renewables_share_energy: {
|
||||
title: 'Renewables, share of primary energy',
|
||||
unit: '%',
|
||||
series: true
|
||||
},
|
||||
low_carbon_share_elec: { title: 'Low-carbon share of electricity', unit: '%', series: true },
|
||||
coal_share_elec: { title: 'Coal, share of electricity', unit: '%', series: true },
|
||||
gas_share_elec: { title: 'Gas, share of electricity', unit: '%', series: true },
|
||||
oil_share_elec: { title: 'Oil, share of electricity', unit: '%', series: true },
|
||||
nuclear_share_elec: { title: 'Nuclear, share of electricity', unit: '%', series: true },
|
||||
hydro_share_elec: { title: 'Hydro, share of electricity', unit: '%', series: true },
|
||||
wind_share_elec: { title: 'Wind, share of electricity', unit: '%', series: true },
|
||||
solar_share_elec: { title: 'Solar, share of electricity', unit: '%', series: true },
|
||||
biofuel_share_elec: { title: 'Bioenergy, share of electricity', unit: '%', series: true },
|
||||
other_renewables_share_elec_exc_biofuel: {
|
||||
title: 'Other renewables, share of electricity',
|
||||
unit: '%'
|
||||
},
|
||||
coal_consumption: { title: 'Coal consumption', unit: 'TWh' },
|
||||
oil_consumption: { title: 'Oil consumption', unit: 'TWh' },
|
||||
gas_consumption: { title: 'Gas consumption', unit: 'TWh' },
|
||||
nuclear_consumption: { title: 'Nuclear consumption', unit: 'TWh' },
|
||||
renewables_consumption: { title: 'Renewables consumption', unit: 'TWh' },
|
||||
net_elec_imports_share_demand: { title: 'Net electricity imports, share of demand', unit: '%' }
|
||||
};
|
||||
|
||||
/** Minimal CSV parser (quoted fields, commas inside quotes). */
|
||||
export function parseCsv(text: string): string[][] {
|
||||
const out: string[][] = [];
|
||||
let row: string[] = [];
|
||||
let field = '';
|
||||
let quoted = false;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const ch = text[i];
|
||||
if (quoted) {
|
||||
if (ch === '"' && text[i + 1] === '"') {
|
||||
field += '"';
|
||||
i++;
|
||||
} else if (ch === '"') quoted = false;
|
||||
else field += ch;
|
||||
} else if (ch === '"') quoted = true;
|
||||
else if (ch === ',') {
|
||||
row.push(field);
|
||||
field = '';
|
||||
} else if (ch === '\n' || ch === '\r') {
|
||||
if (ch === '\r' && text[i + 1] === '\n') i++;
|
||||
row.push(field);
|
||||
out.push(row);
|
||||
row = [];
|
||||
field = '';
|
||||
} else field += ch;
|
||||
}
|
||||
if (field || row.length) {
|
||||
row.push(field);
|
||||
out.push(row);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const regionOf = (iso3: string, entity: string) =>
|
||||
COMPARE_ISO3.includes(iso3)
|
||||
? iso3
|
||||
: entity === 'World'
|
||||
? 'WLD'
|
||||
: entity === 'European Union (27)'
|
||||
? 'EU27'
|
||||
: null;
|
||||
|
||||
export async function owid(opts: { only?: string; job: string }) {
|
||||
await run('owid', opts.job, async (note) => {
|
||||
let total = 0;
|
||||
if (!opts.only || opts.only.includes('maddison')) {
|
||||
try {
|
||||
const [header, ...lines] = parseCsv(await getText(MADDISON_URL));
|
||||
const col = (n: string) => header.indexOf(n);
|
||||
const [e, c, y, g] = [col('entity'), col('code'), col('year'), col('gdp_per_capita')];
|
||||
const rows = lines
|
||||
.filter((l) => l.length > g && l[g] !== '')
|
||||
.map((l) => ({
|
||||
entity: l[e],
|
||||
iso3: l[c] || null,
|
||||
year: Number(l[y]),
|
||||
gdp_pc: Number(l[g])
|
||||
}));
|
||||
if (rows.length < 1000) throw new Error(`only ${rows.length} rows`);
|
||||
await sql.begin(async (tx) => {
|
||||
await tx`delete from stats.owid_maddison`;
|
||||
await insertChunks(tx, 'owid_maddison', rows);
|
||||
});
|
||||
const series: SeriesRow[] = [];
|
||||
const obs: ObsRow[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const r of rows) {
|
||||
const region = regionOf(r.iso3 ?? '', r.entity);
|
||||
if (!region) continue;
|
||||
const id = `owid:maddison:${region}`;
|
||||
if (!seen.has(id)) {
|
||||
seen.add(id);
|
||||
series.push({
|
||||
series_id: id,
|
||||
source_id: 'owid',
|
||||
source_code: `maddison/${r.entity}`,
|
||||
title: 'GDP per capita (Maddison)',
|
||||
unit: 'international-$ at 2011 prices',
|
||||
freq: 'A',
|
||||
region_code: region,
|
||||
theme: 'countries',
|
||||
meta: {
|
||||
url: 'https://ourworldindata.org/grapher/gdp-per-capita-maddison-project-database'
|
||||
}
|
||||
});
|
||||
}
|
||||
obs.push({
|
||||
series_id: id,
|
||||
period: `${String(r.year).padStart(4, '0')}-01-01`,
|
||||
value: r.gdp_pc,
|
||||
status: null
|
||||
});
|
||||
}
|
||||
await writeSeries(series, obs);
|
||||
log(`owid maddison: ${rows.length} rows`);
|
||||
total += rows.length;
|
||||
} catch (error) {
|
||||
note(`maddison: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
if (!opts.only || opts.only.includes('energy')) {
|
||||
try {
|
||||
const [header, ...lines] = parseCsv(await getText(ENERGY_URL));
|
||||
const idx = Object.fromEntries(header.map((h, i) => [h, i]));
|
||||
const keep = Object.keys(ENERGY_COLUMNS).filter((k) => k in idx);
|
||||
const rows = lines
|
||||
.filter((l) => l.length >= header.length - 1 && l[idx.country])
|
||||
.map((l) => {
|
||||
const values: Record<string, number> = {};
|
||||
for (const k of keep)
|
||||
if (l[idx[k]] !== '' && l[idx[k]] != null) values[k] = Number(l[idx[k]]);
|
||||
return {
|
||||
country: l[idx.country],
|
||||
iso3: l[idx.iso_code] || null,
|
||||
year: Number(l[idx.year]),
|
||||
values
|
||||
};
|
||||
})
|
||||
.filter((r) => Object.keys(r.values).length > 0);
|
||||
if (rows.length < 1000) throw new Error(`only ${rows.length} rows`);
|
||||
await sql.begin(async (tx) => {
|
||||
await tx`delete from stats.owid_energy`;
|
||||
await insertChunks(tx, 'owid_energy', rows);
|
||||
});
|
||||
const series: SeriesRow[] = [];
|
||||
const obs: ObsRow[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const r of rows) {
|
||||
const region = regionOf(r.iso3 ?? '', r.country);
|
||||
if (!region) continue;
|
||||
for (const [k, v] of Object.entries(r.values)) {
|
||||
const meta = ENERGY_COLUMNS[k];
|
||||
if (!meta.series) continue;
|
||||
const id = `owid:energy:${k}:${region}`;
|
||||
if (!seen.has(id)) {
|
||||
seen.add(id);
|
||||
series.push({
|
||||
series_id: id,
|
||||
source_id: 'owid',
|
||||
source_code: `energy/${k}/${r.country}`,
|
||||
title: meta.title,
|
||||
unit: meta.unit,
|
||||
freq: 'A',
|
||||
region_code: region,
|
||||
theme: 'energy',
|
||||
meta: { url: 'https://github.com/owid/energy-data' }
|
||||
});
|
||||
}
|
||||
obs.push({ series_id: id, period: `${r.year}-01-01`, value: v, status: null });
|
||||
}
|
||||
}
|
||||
await writeSeries(series, obs);
|
||||
log(`owid energy: ${rows.length} rows`);
|
||||
total += rows.length;
|
||||
} catch (error) {
|
||||
note(`energy: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
return total;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user