Stan Runge
42c9351ebd
All checks were successful
Deploy production / deploy (push) Successful in 1m12s
219 lines
7.5 KiB
TypeScript
219 lines
7.5 KiB
TypeScript
// A9 Our World in Data: Maddison GDP per capita (NL back to 1348) and the OWID energy dataset
|
|
// (energy mix per country). Both are full CSV downloads, replaced on every run.
|
|
import {
|
|
COMPARE_ISO3,
|
|
getText,
|
|
insertChunks,
|
|
log,
|
|
run,
|
|
sql,
|
|
writeSeries,
|
|
type ObsRow,
|
|
type SeriesRow
|
|
} from './common';
|
|
|
|
const MADDISON_URL =
|
|
'https://ourworldindata.org/grapher/gdp-per-capita-maddison-project-database.csv?csvType=full&useColumnShortNames=true';
|
|
const ENERGY_URL = 'https://owid-public.owid.io/data/energy/owid-energy-data.csv';
|
|
|
|
/** OWID energy columns kept (raw) and the ones normalised into series for compared countries. */
|
|
export const ENERGY_COLUMNS: Record<string, { title: string; unit: string; series?: boolean }> = {
|
|
population: { title: 'Population', unit: 'people' },
|
|
energy_per_capita: { title: 'Primary energy per person', unit: 'kWh', series: true },
|
|
primary_energy_consumption: { title: 'Primary energy consumption', unit: 'TWh', series: true },
|
|
per_capita_electricity: { title: 'Electricity generation per person', unit: 'kWh', series: true },
|
|
electricity_generation: { title: 'Electricity generation', unit: 'TWh' },
|
|
carbon_intensity_elec: {
|
|
title: 'Carbon intensity of electricity',
|
|
unit: 'gCO₂/kWh',
|
|
series: true
|
|
},
|
|
fossil_share_energy: { title: 'Fossil fuels, share of primary energy', unit: '%', series: true },
|
|
renewables_share_energy: {
|
|
title: 'Renewables, share of primary energy',
|
|
unit: '%',
|
|
series: true
|
|
},
|
|
low_carbon_share_elec: { title: 'Low-carbon share of electricity', unit: '%', series: true },
|
|
coal_share_elec: { title: 'Coal, share of electricity', unit: '%', series: true },
|
|
gas_share_elec: { title: 'Gas, share of electricity', unit: '%', series: true },
|
|
oil_share_elec: { title: 'Oil, share of electricity', unit: '%', series: true },
|
|
nuclear_share_elec: { title: 'Nuclear, share of electricity', unit: '%', series: true },
|
|
hydro_share_elec: { title: 'Hydro, share of electricity', unit: '%', series: true },
|
|
wind_share_elec: { title: 'Wind, share of electricity', unit: '%', series: true },
|
|
solar_share_elec: { title: 'Solar, share of electricity', unit: '%', series: true },
|
|
biofuel_share_elec: { title: 'Bioenergy, share of electricity', unit: '%', series: true },
|
|
other_renewables_share_elec_exc_biofuel: {
|
|
title: 'Other renewables, share of electricity',
|
|
unit: '%'
|
|
},
|
|
coal_consumption: { title: 'Coal consumption', unit: 'TWh' },
|
|
oil_consumption: { title: 'Oil consumption', unit: 'TWh' },
|
|
gas_consumption: { title: 'Gas consumption', unit: 'TWh' },
|
|
nuclear_consumption: { title: 'Nuclear consumption', unit: 'TWh' },
|
|
renewables_consumption: { title: 'Renewables consumption', unit: 'TWh' },
|
|
net_elec_imports_share_demand: { title: 'Net electricity imports, share of demand', unit: '%' }
|
|
};
|
|
|
|
/** Minimal CSV parser (quoted fields, commas inside quotes). */
|
|
export function parseCsv(text: string, sep = ','): string[][] {
|
|
const out: string[][] = [];
|
|
let row: string[] = [];
|
|
let field = '';
|
|
let quoted = false;
|
|
for (let i = 0; i < text.length; i++) {
|
|
const ch = text[i];
|
|
if (quoted) {
|
|
if (ch === '"' && text[i + 1] === '"') {
|
|
field += '"';
|
|
i++;
|
|
} else if (ch === '"') quoted = false;
|
|
else field += ch;
|
|
} else if (ch === '"') quoted = true;
|
|
else if (ch === sep) {
|
|
row.push(field);
|
|
field = '';
|
|
} else if (ch === '\n' || ch === '\r') {
|
|
if (ch === '\r' && text[i + 1] === '\n') i++;
|
|
row.push(field);
|
|
out.push(row);
|
|
row = [];
|
|
field = '';
|
|
} else field += ch;
|
|
}
|
|
if (field || row.length) {
|
|
row.push(field);
|
|
out.push(row);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
const regionOf = (iso3: string, entity: string) =>
|
|
COMPARE_ISO3.includes(iso3)
|
|
? iso3
|
|
: entity === 'World'
|
|
? 'WLD'
|
|
: entity === 'European Union (27)'
|
|
? 'EU27'
|
|
: null;
|
|
|
|
export async function owid(opts: { only?: string; job: string }) {
|
|
await run('owid', opts.job, async (note) => {
|
|
let total = 0;
|
|
if (!opts.only || opts.only.includes('maddison')) {
|
|
try {
|
|
const [header, ...lines] = parseCsv(await getText(MADDISON_URL));
|
|
const col = (n: string) => header.indexOf(n);
|
|
const [e, c, y, g] = [col('entity'), col('code'), col('year'), col('gdp_per_capita')];
|
|
const rows = lines
|
|
.filter((l) => l.length > g && l[g] !== '')
|
|
.map((l) => ({
|
|
entity: l[e],
|
|
iso3: l[c] || null,
|
|
year: Number(l[y]),
|
|
gdp_pc: Number(l[g])
|
|
}));
|
|
if (rows.length < 1000) throw new Error(`only ${rows.length} rows`);
|
|
await sql.begin(async (tx) => {
|
|
await tx`delete from stats.owid_maddison`;
|
|
await insertChunks(tx, 'owid_maddison', rows);
|
|
});
|
|
const series: SeriesRow[] = [];
|
|
const obs: ObsRow[] = [];
|
|
const seen = new Set<string>();
|
|
for (const r of rows) {
|
|
const region = regionOf(r.iso3 ?? '', r.entity);
|
|
if (!region) continue;
|
|
const id = `owid:maddison:${region}`;
|
|
if (!seen.has(id)) {
|
|
seen.add(id);
|
|
series.push({
|
|
series_id: id,
|
|
source_id: 'owid',
|
|
source_code: `maddison/${r.entity}`,
|
|
title: 'GDP per capita (Maddison)',
|
|
unit: 'international-$ at 2011 prices',
|
|
freq: 'A',
|
|
region_code: region,
|
|
theme: 'countries',
|
|
meta: {
|
|
url: 'https://ourworldindata.org/grapher/gdp-per-capita-maddison-project-database'
|
|
}
|
|
});
|
|
}
|
|
obs.push({
|
|
series_id: id,
|
|
period: `${String(r.year).padStart(4, '0')}-01-01`,
|
|
value: r.gdp_pc,
|
|
status: null
|
|
});
|
|
}
|
|
await writeSeries(series, obs);
|
|
log(`owid maddison: ${rows.length} rows`);
|
|
total += rows.length;
|
|
} catch (error) {
|
|
note(`maddison: ${error instanceof Error ? error.message : String(error)}`);
|
|
}
|
|
}
|
|
if (!opts.only || opts.only.includes('energy')) {
|
|
try {
|
|
const [header, ...lines] = parseCsv(await getText(ENERGY_URL));
|
|
const idx = Object.fromEntries(header.map((h, i) => [h, i]));
|
|
const keep = Object.keys(ENERGY_COLUMNS).filter((k) => k in idx);
|
|
const rows = lines
|
|
.filter((l) => l.length >= header.length - 1 && l[idx.country])
|
|
.map((l) => {
|
|
const values: Record<string, number> = {};
|
|
for (const k of keep)
|
|
if (l[idx[k]] !== '' && l[idx[k]] != null) values[k] = Number(l[idx[k]]);
|
|
return {
|
|
country: l[idx.country],
|
|
iso3: l[idx.iso_code] || null,
|
|
year: Number(l[idx.year]),
|
|
values
|
|
};
|
|
})
|
|
.filter((r) => Object.keys(r.values).length > 0);
|
|
if (rows.length < 1000) throw new Error(`only ${rows.length} rows`);
|
|
await sql.begin(async (tx) => {
|
|
await tx`delete from stats.owid_energy`;
|
|
await insertChunks(tx, 'owid_energy', rows);
|
|
});
|
|
const series: SeriesRow[] = [];
|
|
const obs: ObsRow[] = [];
|
|
const seen = new Set<string>();
|
|
for (const r of rows) {
|
|
const region = regionOf(r.iso3 ?? '', r.country);
|
|
if (!region) continue;
|
|
for (const [k, v] of Object.entries(r.values)) {
|
|
const meta = ENERGY_COLUMNS[k];
|
|
if (!meta.series) continue;
|
|
const id = `owid:energy:${k}:${region}`;
|
|
if (!seen.has(id)) {
|
|
seen.add(id);
|
|
series.push({
|
|
series_id: id,
|
|
source_id: 'owid',
|
|
source_code: `energy/${k}/${r.country}`,
|
|
title: meta.title,
|
|
unit: meta.unit,
|
|
freq: 'A',
|
|
region_code: region,
|
|
theme: 'energy',
|
|
meta: { url: 'https://github.com/owid/energy-data' }
|
|
});
|
|
}
|
|
obs.push({ series_id: id, period: `${r.year}-01-01`, value: v, status: null });
|
|
}
|
|
}
|
|
await writeSeries(series, obs);
|
|
log(`owid energy: ${rows.length} rows`);
|
|
total += rows.length;
|
|
} catch (error) {
|
|
note(`energy: ${error instanceof Error ? error.message : String(error)}`);
|
|
}
|
|
}
|
|
return total;
|
|
});
|
|
}
|