stats step 3: elections, party positions and opinion feeds (schema, collectors, timers); world-opinions data moved into stats.wo_*

scripts/stats/gtrends.ts

@@ -0,0 +1,56 @@
+// A16 Google Trends "trending now" RSS (daily; terms of service: display with attribution).
+// Keeps each day's list per country; a re-run on the same day refreshes rank and traffic.
+import { getText, insertChunks, run, sql } from './common';
+import { decode } from './htmltable';
+
+const GEOS = ['NL', 'US', 'KR'];
+
+/** "200+", "2K+", "1M+" → lower bound as a number. */
+const traffic = (s: string | null) => {
+	const m = s?.replace(/,/g, '').match(/([\d.]+)\s*([KM])?/i);
+	if (!m) return null;
+	return Math.round(Number(m[1]) * (m[2]?.toUpperCase() === 'M' ? 1e6 : m[2] ? 1e3 : 1));
+};
+
+const tag = (xml: string, name: string) => {
+	const m = xml.match(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`));
+	return m ? decode(m[1].replace(/^<!\[CDATA\[|\]\]>$/g, '').trim()) : null;
+};
+
+export async function gtrends(opts: { job: string }) {
+	await run('gtrends', opts.job, async () => {
+		let rows = 0;
+		for (const geo of GEOS) {
+			const xml = await getText(`https://trends.google.com/trending/rss?geo=${geo}`);
+			const today = new Intl.DateTimeFormat('en-CA', { timeZone: 'Europe/Amsterdam' }).format(
+				new Date()
+			);
+			const items = [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m, i) => {
+				const news = [...m[1].matchAll(/<ht:news_item>([\s\S]*?)<\/ht:news_item>/g)]
+					.map((n) => ({
+						title: tag(n[1], 'ht:news_item_title'),
+						source: tag(n[1], 'ht:news_item_source'),
+						url: tag(n[1], 'ht:news_item_url')
+					}))
+					.slice(0, 3);
+				const pub = tag(m[1], 'pubDate');
+				return {
+					geo,
+					date: today,
+					rank: i + 1,
+					query: tag(m[1], 'title') ?? '',
+					approx_traffic: traffic(tag(m[1], 'ht:approx_traffic')),
+					pub_date: pub ? new Date(pub) : null,
+					news: JSON.stringify(news)
+				};
+			});
+			const unique = [...new Map(items.filter((x) => x.query).map((x) => [x.query, x])).values()];
+			await sql.begin(async (tx) => {
+				await tx`delete from stats.gt_trending where geo = ${geo} and date = ${today}`;
+				await insertChunks(tx, 'gt_trending', unique);
+			});
+			rows += unique.length;
+		}
+		return rows;
+	});
+}