Stan Runge
42c9351ebd
All checks were successful
Deploy production / deploy (push) Successful in 1m12s
87 lines
2.7 KiB
TypeScript
87 lines
2.7 KiB
TypeScript
// A16 GDELT DOC 2.0 "timelinevol": daily share of monitored news coverage (by source country)
|
|
// that mentions a GKG theme, last 30 days, upserted nightly. GDELT asks for at most one request
|
|
// every 5 seconds, so the loop is deliberately slow (~12 minutes). GDELT data is open, cite GDELT.
|
|
import { http, insertChunks, run, sql } from './common';
|
|
|
|
export const GDELT_COUNTRIES: Record<string, string> = {
|
|
netherlands: 'NLD',
|
|
unitedstates: 'USA',
|
|
southkorea: 'KOR',
|
|
germany: 'DEU',
|
|
france: 'FRA',
|
|
unitedkingdom: 'GBR',
|
|
belgium: 'BEL',
|
|
japan: 'JPN',
|
|
india: 'IND',
|
|
brazil: 'BRA'
|
|
};
|
|
export const GDELT_THEMES = [
|
|
'ECON_INFLATION',
|
|
'ECON_HOUSING_PRICES',
|
|
'UNEMPLOYMENT',
|
|
'IMMIGRATION',
|
|
'REFUGEES',
|
|
'ENV_CLIMATECHANGE',
|
|
'TERROR',
|
|
'PROTEST',
|
|
'CORRUPTION',
|
|
'ELECTION',
|
|
'MILITARY',
|
|
'HEALTH_PANDEMIC'
|
|
];
|
|
|
|
export async function gdelt(opts: { job: string; timespan?: string }) {
|
|
await run('gdelt', opts.job, async (note) => {
|
|
let rows = 0;
|
|
const bad = new Set<string>();
|
|
for (const [name, iso3] of Object.entries(GDELT_COUNTRIES))
|
|
for (const theme of GDELT_THEMES) {
|
|
if (bad.has(theme)) continue;
|
|
const q = encodeURIComponent(`theme:${theme} sourcecountry:${name}`);
|
|
const url = `https://api.gdeltproject.org/api/v2/doc/doc?query=${q}&mode=timelinevol&format=json×pan=${opts.timespan ?? '30d'}`;
|
|
let text: string;
|
|
try {
|
|
text = await (await http(url, {}, 3)).text();
|
|
} catch (error) {
|
|
note(`${theme}/${name}: ${error instanceof Error ? error.message : error}`);
|
|
continue;
|
|
}
|
|
let data: { timeline?: { data: { date: string; value: number }[] }[] };
|
|
try {
|
|
data = JSON.parse(text);
|
|
} catch {
|
|
// GDELT answers invalid queries with a plain-text message.
|
|
note(`${theme}: ${text.slice(0, 120)}`);
|
|
if (/theme|invalid/i.test(text)) bad.add(theme);
|
|
continue;
|
|
}
|
|
const points = (data.timeline?.[0]?.data ?? []).map((p) => ({
|
|
date: `${p.date.slice(0, 4)}-${p.date.slice(4, 6)}-${p.date.slice(6, 8)}`,
|
|
country: iso3,
|
|
theme,
|
|
vol_pct: p.value
|
|
}));
|
|
// Hourly resolution (short timespans): average per day.
|
|
const byDay = new Map<string, { sum: number; n: number; row: (typeof points)[number] }>();
|
|
for (const p of points) {
|
|
const d = byDay.get(p.date) ?? { sum: 0, n: 0, row: p };
|
|
d.sum += p.vol_pct;
|
|
d.n++;
|
|
byDay.set(p.date, d);
|
|
}
|
|
const daily = [...byDay.values()].map((d) => ({ ...d.row, vol_pct: d.sum / d.n }));
|
|
if (!daily.length) continue;
|
|
await sql.begin(async (tx) => {
|
|
await insertChunks(
|
|
tx,
|
|
'gdelt_theme_vol',
|
|
daily,
|
|
'on conflict (date, country, theme) do update set vol_pct = excluded.vol_pct'
|
|
);
|
|
});
|
|
rows += daily.length;
|
|
}
|
|
return rows;
|
|
});
|
|
}
|